reading unknown length file in buffer chunks in c
NickName:t_yand Ask DateTime:2018-09-27T11:02:57

reading unknown length file in buffer chunks in c

I'm trying to read unknown length binary file into buffer chunks without using the functions like lseek(),fseek.

  1. I have used struct buffer that has 1024 bytes at once. when reading file larger than 1012 bytes it will allocate several buffers. However, when it encounters the last chunk it will definitely have less or equal to 1024 bytes. Thus, i try to count the length of the last chunk so that I can read last chunk up until the eof but i am kind of confused with how to implement this.

Thanks in advance.

#include <stdio.h>
#include <stdlib.h>

typedef struct Buffer{
  unsigned char data[1012];
  struct Buffer *next; //12 bytes
}Buffer;

void mymemcpy(void *dest, void *src, size_t length){
  Buffer *buffer_toFill = (Buffer *)dest;
  Buffer *buffer_toAdd = (Buffer *)src;
  int a = 0; 
  for(int i = 0; i < length; i++){
    buffer_toFill->data[i] = buffer_toAdd->data[i];
  }
}

Buffer* add_buffer_front(Buffer *head, Buffer *read_buffer, int size){
  Buffer *new_buffer = malloc(sizeof(Buffer));
  mymemcpy(new_buffer, read_buffer, size);
  if(head != NULL){
    new_buffer->next = head;
  }
  return new_buffer;
}

void display_List(Buffer *head, size_t length){
  Buffer *current = head;
  while(current != NULL){
    for(int i = 0; i < length; i++){
      printf("%02X",(unsigned)current->data[i]); //this shows different value compare with  xxd <filename>
      //printf("%c", current->data[i]);  
    }
    Buffer *prev = current;
    free(prev);
    current = current->next;
  }
}

int main(int argc, char **argv){
  FILE *fd;
  Buffer *head_buffer = NULL;
  int file_length = 0;
  int eof_int = 1;
  if(argc != 2){
    printf("Usage: readFile <filename>\n");
    return 1; 
  }

  fd = fopen(argv[1], "rb");

  while(eof_int != 0){ 
    Buffer *new_buffer = malloc(sizeof(Buffer));
    eof_int = fread(new_buffer, sizeof(Buffer)-12, 1, fd);
    if(eof_int == 0){ 
      //size_t length
      //
      //
      head_buffer = add_buffer_front(head_buffer, new_buffer, length);
      file_length += length;
    }else{
      head_buffer = add_buffer_front(head_buffer, new_buffer, (sizeof(new_buffer->data)));
      file_length += (sizeof(new_buffer->data));
    }
  }
  display_List(head_buffer, file_length);
  fclose(fd);
  return 0;
}

Copyright Notice:Content Author:「t_yand」,Reproduced under the CC 4.0 BY-SA copyright license with a link to the original source and this disclaimer.
Link to original article:https://stackoverflow.com/questions/52528783/reading-unknown-length-file-in-buffer-chunks-in-c

Answers
Leonard 2018-09-27T03:25:23

You have several problems.\n\n(1) fread returns the number of items read, but it will not return an eof indication. You need to call feof(stream*) to find out if you've reached the end of file.\n\n(2) You are saying your next pointer is 12 bytes. This is a very dangerous assumption. Prefer to read the 1012 bytes you've allocated to the data struct. In all likelihood you are currently printing stuff that wasn't read in, but is just uninitialized memory.\n\n(3) Use the return value from fread to decide how much memory to copy.",


Ed Heal 2018-09-27T04:04:23

Please see the comments in the code below - also consider changing the 1012 to use a #define. \n\n#include <stdio.h>\n#include <stdlib.h>\n\ntypedef struct Buffer{\n unsigned char data[1012];\n struct Buffer *next; //12 bytes\n}Buffer;\n\n// Create a structure to store stuff about a file\n\ntypedef struct {\n Buffer *head;\n Buffer *tail;\n size_t length;\n} MyFile;\n\n/*\nvoid mymemcpy(void *dest, void *src, size_t length){\n Buffer *buffer_toFill = (Buffer *)dest;\n Buffer *buffer_toAdd = (Buffer *)src;\n int a = 0; \n for(int i = 0; i < length; i++){\n buffer_toFill->data[i] = buffer_toAdd->data[i];\n }\n}\n\nBuffer* add_buffer_front(Buffer *head, Buffer *read_buffer, int size){\n Buffer *new_buffer = malloc(sizeof(Buffer));\n mymemcpy(new_buffer, read_buffer, size);\n if(head != NULL){\n new_buffer->next = head;\n }\n return new_buffer;\n}\n\n*/\n\n// Lets make this easier - The buffer has already been \"malloced\" once - why do it again\n\n// And why are you reversing the file\n\n// Perhaps \n\nvoid add_buffer(Buffer *to_be_added, MyFile *file, size_t extra_length) {\n if (file->tail) { // We have one item in the list\n file->tail->next = to_be_added;\n } else { // First buffer!\n file-> head = to_be_added;\n file-> tail = to_be_added;\n }\n to_be_added->next = NULL; // This is always the case as it is the last one\n file->length += extra_length;\n}\n\n/*\nvoid display_List(Buffer *head, size_t length){\n Buffer *current = head;\n while(current != NULL){\n for(int i = 0; i < length; i++){\n printf(\"%02X\",(unsigned)current->data[i]); //this shows different value compare with xxd <filename>\n //printf(\"%c\", current->data[i]); \n }\n Buffer *prev = current;\n free(prev);\n current = current->next;\n }\n}\n\n*/\n\n// Instead pass in the new structure\n\nvoid display_list(MyFile *file) {\n size_t contents_left = file -> length;\n Buffer * current = file -> head;\n while (current) {\n // At most each chunk has 1012 bytes - Check for that\n size_t chunk_length = contents_left > 1012 ? 1012 : contents_left;\n for(int i = 0; i <chunk_length ; i++){\n printf(\"%02X\",(unsigned)current->data[i]);\n }\n current = current -> next;\n }\n}\n\n\n}\nint main(int argc, char **argv){\n FILE *fd;\n MyFile read_file;\n read_file.head = NULL;\n read_file.tail = NULL;\n read_file.length = 0;\n\n Buffer *head_buffer = NULL;\n int file_length = 0;\n int eof_int = 1;\n if(argc != 2){\n printf(\"Usage: readFile <filename>\\n\");\n return 1; \n }\n\n fd = fopen(argv[1], \"rb\");\n\n // Check fd\n if (fd == NULL) {\n // error stuff\n return EXIT_FAILURE; // Look up the include for this\n }\n while(eof_int != 0){ \n Buffer *new_buffer = malloc(sizeof(Buffer));\n eof_int = fread(new_buffer->data, 1012, 1, fd); // Do not make assumptions on the size of a pointer and store it in the correct location\n if(eof_int == 0) { // Read nothing\n free(new_buffer); // We was too optimistic! Did Not need this in the end \n break;\n } else {\n add_buffer(&read_file, new_buffer, eof_int);\n }\n }\n display_List(&read_file);\n fclose(fd);\n return 0;\n}\n",


More about “reading unknown length file in buffer chunks in c” related questions

reading unknown length file in buffer chunks in c

I'm trying to read unknown length binary file into buffer chunks without using the functions like lseek(),fseek. I have used struct buffer that has 1024 bytes at once. when reading file larger tha...

Show Detail

Reading chunks of data from file (C)

My binary file contains chunks of data in the following format: 0xAA ... variable length of bytes ... 0XFF Is there a good way to read these chunks of data directly into a buffer, instead of rea...

Show Detail

Reading and writing file in number of Byte Buffer chunks of smaller length

I am trying to read file in ByteBuffer chunks of fixed length and then store it to a list of ByteBuffer and then after some operations read those ByteBuffer chunks in a sequential order to reconstr...

Show Detail

Reading from text file using buffer and multithreading in C

Part of a project I am working on requires me to read from a text file of unknown size and put the data into a 10KB buffer. I have to read the file into this buffer in 2KB chunks. I must have 4 thr...

Show Detail

Upload file in chunks with unknown length

Can you give me an example (html or java) to implement the upload file in chunks. How can I upload a file If the length is unknown at the time of the request? I found this way https://developers.go...

Show Detail

reading and writing in chunks on linux using c

I have a ASCII file where every line contains a record of variable length. For example Record-1:15 characters Record-2:200 characters Record-3:500 characters ... ... Record-n: X characters As the...

Show Detail

Read large txt file and use it in chunks

I'm newbie in C# but I've made some research for reading/writing large txt files, the biggest might be 8GB but if it is too much I will consider split it maybe to 1GB. It must be fast up to 30MByte...

Show Detail

Copying unknown length buffer to a fixed size buffer in C

How can we copy an unknown size buffer into a fixed buffer in c? For one of my functions, I'm trying to copy an unknown size buffer into a fixed buffer (of size 1024 bytes). The fixed size buffer is

Show Detail

Append chunks of bytes to a buffer and save the buffer to file

I have a simple c# application that records the screen with ffmpeg and redirects the standardoutput which is sent in chunks to a node.js server, but i am having some trouble with the file getting c...

Show Detail

Splitting a file into chunks

I'm trying to split large files (3gb+) into chunks of 100mb, then sending those chunks through HTTP. For testing, i'm working on a 29 mb file, size: 30380892, size on disk: 30384128 (so there is no...

Show Detail