blocks: Fix file_source offset/length to properly access large files.

author: Ron Economos <w6rz@comcast.net> 2018-12-21 23:11:11 -0800
committer: Andrej Rode <mail@andrejro.de> 2019-02-14 11:17:48 +0100
commit: addf04b9c51779a423a4795f02c941eaff5a33a1 (patch)
tree: 90c4558a18e1b075e8586a0795a40365dc287f8b
parent: b17a9b3f0672318cb6fa240d14aed14ffffebe69 (diff)
4 files changed, 32 insertions, 48 deletions
diff --git a/gr-blocks/include/gnuradio/blocks/file_source.h b/gr-blocks/include/gnuradio/blocks/file_source.h
index cd73c97d02..007267a7c9 100644
--- a/gr-blocks/include/gnuradio/blocks/file_source.h
+++ b/gr-blocks/include/gnuradio/blocks/file_source.h
@@ -62,7 +62,7 @@ namespace gr {
        * \param len     produce only items [offset, offset+len)
        */
       static sptr make(size_t itemsize, const char *filename, bool repeat = false,
-                       size_t offset = 0, size_t len = 0);
+                       uint64_t offset = 0, uint64_t len = 0);
 
       /*!
        * \brief seek file to \p seek_point relative to \p whence
@@ -70,7 +70,7 @@ namespace gr {
        * \param seek_point      sample offset in file
        * \param whence  one of SEEK_SET, SEEK_CUR, SEEK_END (man fseek)
        */
-      virtual bool seek(long seek_point, int whence) = 0;
+      virtual bool seek(int64_t seek_point, int whence) = 0;
 
       /*!
        * \brief Opens a new file.
@@ -80,7 +80,7 @@ namespace gr {
        * \param offset  begin this many items into file
        * \param len     produce only items [offset, offset+len)
        */
-      virtual void open(const char *filename, bool repeat, size_t offset = 0, size_t len = 0) = 0;
+      virtual void open(const char *filename, bool repeat, uint64_t offset = 0, uint64_t len = 0) = 0;
 
       /*!
        * \brief Close the file handle.
diff --git a/gr-blocks/lib/CMakeLists.txt b/gr-blocks/lib/CMakeLists.txt
index cfb728ce22..dda5c395cb 100644
--- a/gr-blocks/lib/CMakeLists.txt
+++ b/gr-blocks/lib/CMakeLists.txt
@@ -1,4 +1,4 @@
-# Copyright 2012-2014 Free Software Foundation, Inc.
+# Copyright 2012-2014,2018 Free Software Foundation, Inc.
 #
 # This file is part of GNU Radio
 #
@@ -199,6 +199,8 @@ list(APPEND gr_blocks_sources
     wavfile_source_impl.cc
 )
 
+set_source_files_properties(file_source_impl.cc PROPERTIES COMPILE_FLAGS -D_FILE_OFFSET_BITS=64)
+
 if(ENABLE_GR_CTRLPORT)
 list(APPEND blocks_ctrlport_sources
   ctrlport_probe_c_impl.cc
diff --git a/gr-blocks/lib/file_source_impl.cc b/gr-blocks/lib/file_source_impl.cc
index 6f4f8fde26..3f7b844e03 100644
--- a/gr-blocks/lib/file_source_impl.cc
+++ b/gr-blocks/lib/file_source_impl.cc
@@ -34,34 +34,26 @@
 #include <stdexcept>
 #include <stdio.h>
 
-// win32 (mingw/msvc) specific
-#ifdef HAVE_IO_H
-#include <io.h>
-#endif
-#ifdef O_BINARY
-#define OUR_O_BINARY O_BINARY
-#else
-#define OUR_O_BINARY 0
-#endif
-// should be handled via configure
-#ifdef O_LARGEFILE
-#define OUR_O_LARGEFILE O_LARGEFILE
+#ifdef _MSC_VER
+#define GR_FSEEK _fseeki64
+#define GR_FTELL _ftelli64
 #else
-#define OUR_O_LARGEFILE 0
+#define GR_FSEEK fseeko
+#define GR_FTELL ftello
 #endif
 
 namespace gr {
   namespace blocks {
 
     file_source::sptr file_source::make(size_t itemsize, const char *filename, bool repeat,
-                                        size_t start_offset_items, size_t length_items)
+                                        uint64_t start_offset_items, uint64_t length_items)
     {
       return gnuradio::get_initial_sptr
         (new file_source_impl(itemsize, filename, repeat, start_offset_items, length_items));
     }
 
     file_source_impl::file_source_impl(size_t itemsize, const char *filename, bool repeat,
-                                       size_t start_offset_items, size_t length_items)
+                                       uint64_t start_offset_items, uint64_t length_items)
       : sync_block("file_source",
                    io_signature::make(0, 0, 0),
                    io_signature::make(1, 1, itemsize)),
@@ -87,7 +79,7 @@ namespace gr {
     }
 
     bool
-    file_source_impl::seek(long seek_point, int whence)
+    file_source_impl::seek(int64_t seek_point, int whence)
     {
       seek_point += d_start_offset_items;
 
@@ -105,45 +97,35 @@ namespace gr {
         return 0;
       }
 
-      if ((seek_point < (long)d_start_offset_items)
-          || (seek_point > (long)(d_start_offset_items+d_length_items-1))) {
+      if ((seek_point < (int64_t)d_start_offset_items)
+          || (seek_point > (int64_t)(d_start_offset_items+d_length_items-1))) {
         GR_LOG_WARN(d_logger, "bad seek point");
         return 0;
       }
-
-      return fseek((FILE*)d_fp, seek_point * d_itemsize, SEEK_SET) == 0;
+      return GR_FSEEK((FILE*)d_fp, seek_point * d_itemsize, SEEK_SET) == 0;
     }
 
 
     void
     file_source_impl::open(const char *filename, bool repeat,
-                           size_t start_offset_items, size_t length_items)
+                           uint64_t start_offset_items, uint64_t length_items)
     {
       // obtain exclusive access for duration of this function
       gr::thread::scoped_lock lock(fp_mutex);
 
-      int fd;
-
-      // we use "open" to use to the O_LARGEFILE flag
-      if((fd = ::open(filename, O_RDONLY | OUR_O_LARGEFILE | OUR_O_BINARY)) < 0) {
-        GR_LOG_ERROR(d_logger, boost::format("%s: %s") % filename % strerror(errno));
-        throw std::runtime_error("can't open file");
-      }
-
       if(d_new_fp) {
         fclose(d_new_fp);
         d_new_fp = 0;
       }
 
-      if((d_new_fp = fdopen (fd, "rb")) == NULL) {
+      if((d_new_fp = fopen (filename, "rb")) == NULL) {
         GR_LOG_ERROR(d_logger, boost::format("%s: %s") % filename % strerror(errno));
-        ::close(fd);    // don't leak file descriptor if fdopen fails
         throw std::runtime_error("can't open file");
       }
 
       //Check to ensure the file will be consumed according to item size
-      fseek(d_new_fp, 0, SEEK_END);
-      int file_size = ftell(d_new_fp);
+      GR_FSEEK(d_new_fp, 0, SEEK_END);
+      uint64_t file_size = GR_FTELL(d_new_fp);
 
       // Make sure there will be at least one item available
       if ((file_size / d_itemsize) < (start_offset_items+1)) {
@@ -157,7 +139,7 @@ namespace gr {
         throw std::runtime_error("file is too small");
       }
 
-      size_t items_available = (file_size / d_itemsize - start_offset_items);
+      uint64_t items_available = (file_size / d_itemsize - start_offset_items);
 
       // If length is not specified, use the remainder of the file. Check alignment at end.
       if (length_items == 0) {
@@ -174,7 +156,7 @@ namespace gr {
       }
 
       // Rewind to start offset
-      fseek(d_new_fp, start_offset_items * d_itemsize, SEEK_SET);
+      GR_FSEEK(d_new_fp, start_offset_items * d_itemsize, SEEK_SET);
 
       d_updated = true;
       d_repeat = repeat;
@@ -224,7 +206,7 @@ namespace gr {
                            gr_vector_void_star &output_items)
     {
       char *o = (char*)output_items[0];
-      int size = noutput_items;
+      uint64_t size = noutput_items;
 
       do_update();       // update d_fp is reqd
       if(d_fp == NULL)
@@ -245,7 +227,7 @@ namespace gr {
           d_file_begin = false;
         }
 
-        size_t nitems_to_read = std::min((size_t)size, d_items_remaining);
+        uint64_t nitems_to_read = std::min(size, d_items_remaining);
 
         // Since the bounds of the file are known, unexpected nitems is an error
         if (nitems_to_read != fread(o, d_itemsize, nitems_to_read, (FILE*)d_fp))
@@ -260,7 +242,7 @@ namespace gr {
 
           // Repeat: rewind and request tag
           if (d_repeat) {
-            fseek(d_fp, d_start_offset_items * d_itemsize, SEEK_SET);
+            GR_FSEEK(d_fp, d_start_offset_items * d_itemsize, SEEK_SET);
             d_items_remaining = d_length_items;
             if (d_add_begin_tag != pmt::PMT_NIL) {
               d_file_begin = true;
diff --git a/gr-blocks/lib/file_source_impl.h b/gr-blocks/lib/file_source_impl.h
index 6de3fc0d16..4866ac86a3 100644
--- a/gr-blocks/lib/file_source_impl.h
+++ b/gr-blocks/lib/file_source_impl.h
@@ -33,9 +33,9 @@ namespace gr {
     {
     private:
       size_t d_itemsize;
-      size_t d_start_offset_items;
-      size_t d_length_items;
-      size_t d_items_remaining;
+      uint64_t d_start_offset_items;
+      uint64_t d_length_items;
+      uint64_t d_items_remaining;
       FILE *d_fp;
       FILE *d_new_fp;
       bool d_repeat;
@@ -51,11 +51,11 @@ namespace gr {
 
     public:
       file_source_impl(size_t itemsize, const char *filename, bool repeat,
-                       size_t offset, size_t len);
+                       uint64_t offset, uint64_t len);
       ~file_source_impl();
 
-      bool seek(long seek_point, int whence);
-      void open(const char *filename, bool repeat, size_t offset, size_t len);
+      bool seek(int64_t seek_point, int whence);
+      void open(const char *filename, bool repeat, uint64_t offset, uint64_t len);
       void close();
 
       int work(int noutput_items,
author	Ron Economos <w6rz@comcast.net>	2018-12-21 23:11:11 -0800
committer	Andrej Rode <mail@andrejro.de>	2019-02-14 11:17:48 +0100
commit	addf04b9c51779a423a4795f02c941eaff5a33a1 (patch)
tree	90c4558a18e1b075e8586a0795a40365dc287f8b
parent	b17a9b3f0672318cb6fa240d14aed14ffffebe69 (diff)