View Issue Details

IDProjectCategoryView StatusLast Update
0034281FPCUtilitiespublic2018-11-04 13:04
ReporterRossAssigned ToMichael Van Canneyt 
PrioritynormalSeverityminorReproducibilityalways
Status resolvedResolutionfixed 
PlatformWin64OSWindowsOS Version7
Product Version3.0.4Product Build 
Target Version3.2.0Fixed in Version3.3.1 
Summary0034281: Zipping large file 7GB results in stream read error
DescriptionCode throws stream read error while zipping a 7 GB file with paszlib TZipper.
The resulting zip file when opened with 7Zip shows
Size: 4294967295
Packed Size: 222909700

This version is supposed to support Zip64.

Steps To ReproduceZip one 7 gigabyte file.
TagsNo tags attached.
Fixed in Revision40215
FPCOldBugId
FPCTarget
Attached Files
  • tezipper.pas (2,037 bytes)
    { https://bugs.freepascal.org/view.php?id=34281 }
    program tezipper;  {test TZipper class}
    
    {$mode objfpc}
    uses zipper, classes, sysutils;
    
    procedure filldata(data : pbyte; size : dword);
    var iA : dword;
        selfData : pbyte;
    begin
         selfData := pbyte(@fillData); {code}
         for iA:=0 to size-1 do
         begin
    
              data^:=selfData^;
              inc(data);
              inc(selfData);
         end;
    end;
    
    procedure createFileWithSize(fname:string;size : qword);
    var
        totalSize : qword;
        tmpData : pbyte;
        iA : dword;
        bufSize,wrBufSize, wrCount : dword;
        fn : file;
    
    
    begin
         totalSize:=0;
         bufSize:=2048;
         wrBufSize:=bufSize;
         getMem(tmpData, bufSize);
    
         assign(fn, fname);
         rewrite(fn, 1);
    
         fillData ( tmpData, bufSize);
         while totalSize < size do
         begin
              if totalSize+wrBufSize> size then
                   wrBufSize:=size-totalSize;
              blockWrite(fn,tmpData^, wrBufSize, wrCount);
              totalSize:=totalSize+wrCount;
              if wrCount < wrBufSize then break;
         end;
    
         close(fn);
    
         freeMem(tmpData, bufSize);
         writeln('File ',fname,' created. Size ',totalSize,'  (',totalSize / 1024 / 1024 / 1024:5:4,' Gb)');
    end;
    
    procedure zip(zipName, fname : string);
    var
      OurZipper: TZipper;
    begin
    
      OurZipper := TZipper.Create;
      try
        OurZipper.FileName := zipName;
    
         OurZipper.Entries.AddFileEntry(fname, fname);
         OurZipper.ZipAllFiles;
      finally
        OurZipper.Free;
      end;
    
    
    end;
    
    procedure unzip(zipName : string);
    var
      UnZipper: TUnZipper;
    begin
      UnZipper := TUnZipper.Create;
      try
        UnZipper.FileName := zipName;
        UnZipper.OutputPath := 'temp';
        UnZipper.Examine;
        UnZipper.UnZipAllFiles;
      finally
        UnZipper.Free;
      end;
    end;
    
    begin
    	writeln;
         writeln('Create mega size file');
         createFileWithSize('mega7gb.tmp',4 * 1024*1024*1024+1024);
    	write('Wait! Zipping.. ');
    	zip ('zz.zip', 'mega7gb.tmp');
         writeln;
    
         write('Wait! UnZipping..');
    	unzip('zz.zip');
         writeln;
    
    	writeln('Zip test finished.');
    end.
    
    tezipper.pas (2,037 bytes)
  • zipper_7gb.patch (3,077 bytes)
    Index: zipper.pp
    ===================================================================
    --- zipper.pp	(revision 40017)
    +++ zipper.pp	(working copy)
    @@ -1609,6 +1609,8 @@
           begin
             Compressed_Size := FZip.Size;
             LocalZip64Fld.Compressed_Size := 0;
    +        if LocalZip64Fld.Original_Size > 0 then
    +          IsZip64 := true;
           end;
           if AZipVersionReqd > Extract_Version_Reqd then
             Extract_Version_Reqd := AZipVersionReqd;
    @@ -1623,7 +1625,6 @@
       FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
       if IsZip64 then
       begin
    -    LocalZip64ExtHdr.Header_ID:=ZIP64_HEADER_ID;
         FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapEDFH{$ENDIF}(LocalZip64ExtHdr),SizeOf(LocalZip64ExtHdr));
         FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapZ64EIF{$ENDIF}(LocalZip64Fld),SizeOf(LocalZip64Fld));
       end;
    @@ -1682,7 +1683,7 @@
                 end;
               end;
           // Move past extra fields
    -      FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
    +      //FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
           end;
         SavePos := FOutStream.Position;
         FillChar(CentralHdr,SizeOf(CentralHdr),0);
    @@ -1732,9 +1733,16 @@
           else
             Local_Header_Offset := HdrPos;
           end;
    +
    +      if IsZip64 then
    +      begin
    +          Extra_Field_Length:=SizeOf(LocalZip64ExtHdr)+SizeOf(LocalZip64Fld);
    +      end else Extra_Field_Length :=0;
    +
         FOutStream.Seek(0,soEnd);
         FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapCFH{$ENDIF}(CentralHdr),SizeOf(CentralHdr));
         FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
    +
         if IsZip64 then
           begin
           FOutStream.Seek(0,soEnd);
    @@ -1744,7 +1752,7 @@
     
         Inc(ACount);
         // Move past compressed file data to next header:
    -    if Iszip64 then
    +    if LocalZip64Fld.Compressed_Size > 0 then
           FOutStream.Seek(SavePos + LocalZip64Fld.Compressed_Size,soBeginning)
         else
           FOutStream.Seek(SavePos + LocalHdr.Compressed_Size,soBeginning);
    @@ -2093,7 +2101,7 @@
     Var
       Path: RawByteString;
       OldDirectorySeparators: set of char;
    -  
    +
     Begin
       { the default RTL behavior is broken on Unix platforms
         for Windows compatibility: it allows both '/' and '\'
    @@ -2521,7 +2529,10 @@
               end;
             end;
           // Move past extra fields and file comment to next header
    -      FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
    +      if File_Comment_Length > 0 then
    +          FZipStream.Seek(File_Comment_Length,soFromCurrent);
    +      // this doesn't work properly when zip file size is over 4Gb, so commented off
    +      //FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
           end;
         end;
     end;
    @@ -2605,7 +2616,7 @@
               OnProgress:=Self.OnProgress;
               OnPercent:=Self.OnPercent;
               DeCompress;
    -          Self.FTotPos := FTotPos; 
    +          Self.FTotPos := FTotPos;
               if Item.CRC32 <> Crc32Val then
                 raise EZipError.CreateFmt(SErrInvalidCRC,[Item.ArchiveFileName]);
             Finally
    
    zipper_7gb.patch (3,077 bytes)
  • 7gb-zipper.pp.patch (3,170 bytes)
    Index: packages/paszlib/src/zipper.pp
    ===================================================================
    --- packages/paszlib/src/zipper.pp	(revision 40029)
    +++ packages/paszlib/src/zipper.pp	(working copy)
    @@ -1609,6 +1609,8 @@
           begin
             Compressed_Size := FZip.Size;
             LocalZip64Fld.Compressed_Size := 0;
    +        if LocalZip64Fld.Original_Size > 0 then
    +          IsZip64 := true;
           end;
           if AZipVersionReqd > Extract_Version_Reqd then
             Extract_Version_Reqd := AZipVersionReqd;
    @@ -1623,7 +1625,6 @@
       FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
       if IsZip64 then
       begin
    -    LocalZip64ExtHdr.Header_ID:=ZIP64_HEADER_ID;
         FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapEDFH{$ENDIF}(LocalZip64ExtHdr),SizeOf(LocalZip64ExtHdr));
         FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapZ64EIF{$ENDIF}(LocalZip64Fld),SizeOf(LocalZip64Fld));
       end;
    @@ -1682,7 +1683,7 @@
                 end;
               end;
           // Move past extra fields
    -      FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
    +      //FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
           end;
         SavePos := FOutStream.Position;
         FillChar(CentralHdr,SizeOf(CentralHdr),0);
    @@ -1732,9 +1733,16 @@
           else
             Local_Header_Offset := HdrPos;
           end;
    +
    +      if IsZip64 then
    +      begin
    +          LocalHdr.Extra_Field_Length:=SizeOf(LocalZip64ExtHdr)+SizeOf(LocalZip64Fld);
    +      end else LocalHdr.Extra_Field_Length :=0;
    +
         FOutStream.Seek(0,soEnd);
         FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapCFH{$ENDIF}(CentralHdr),SizeOf(CentralHdr));
         FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
    +
         if IsZip64 then
           begin
           FOutStream.Seek(0,soEnd);
    @@ -1744,7 +1752,7 @@
     
         Inc(ACount);
         // Move past compressed file data to next header:
    -    if Iszip64 then
    +    if LocalZip64Fld.Compressed_Size > 0 then
           FOutStream.Seek(SavePos + LocalZip64Fld.Compressed_Size,soBeginning)
         else
           FOutStream.Seek(SavePos + LocalHdr.Compressed_Size,soBeginning);
    @@ -2093,7 +2101,7 @@
     Var
       Path: RawByteString;
       OldDirectorySeparators: set of char;
    -  
    +
     Begin
       { the default RTL behavior is broken on Unix platforms
         for Windows compatibility: it allows both '/' and '\'
    @@ -2521,7 +2529,10 @@
               end;
             end;
           // Move past extra fields and file comment to next header
    -      FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
    +      if File_Comment_Length > 0 then
    +          FZipStream.Seek(File_Comment_Length,soFromCurrent);
    +      // this doesn't work properly when zip file size is over 4Gb, so commented off
    +      //FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
           end;
         end;
     end;
    @@ -2605,7 +2616,7 @@
               OnProgress:=Self.OnProgress;
               OnPercent:=Self.OnPercent;
               DeCompress;
    -          Self.FTotPos := FTotPos; 
    +          Self.FTotPos := FTotPos;
               if Item.CRC32 <> Crc32Val then
                 raise EZipError.CreateFmt(SErrInvalidCRC,[Item.ArchiveFileName]);
             Finally
    
    7gb-zipper.pp.patch (3,170 bytes)
  • 7gb_zipper_v3.patch (3,162 bytes)
    Index: packages/paszlib/src/zipper.pp
    ===================================================================
    --- packages/paszlib/src/zipper.pp	(revision 40057)
    +++ packages/paszlib/src/zipper.pp	(working copy)
    @@ -1609,6 +1609,8 @@
           begin
             Compressed_Size := FZip.Size;
             LocalZip64Fld.Compressed_Size := 0;
    +        if LocalZip64Fld.Original_Size > 0 then
    +          IsZip64 := true;
           end;
           if AZipVersionReqd > Extract_Version_Reqd then
             Extract_Version_Reqd := AZipVersionReqd;
    @@ -1623,7 +1625,6 @@
       FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
       if IsZip64 then
       begin
    -    LocalZip64ExtHdr.Header_ID:=ZIP64_HEADER_ID;
         FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapEDFH{$ENDIF}(LocalZip64ExtHdr),SizeOf(LocalZip64ExtHdr));
         FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapZ64EIF{$ENDIF}(LocalZip64Fld),SizeOf(LocalZip64Fld));
       end;
    @@ -1682,7 +1683,7 @@
                 end;
               end;
           // Move past extra fields
    -      FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
    +      //FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
           end;
         SavePos := FOutStream.Position;
         FillChar(CentralHdr,SizeOf(CentralHdr),0);
    @@ -1732,9 +1733,16 @@
           else
             Local_Header_Offset := HdrPos;
           end;
    +
    +      if IsZip64 then
    +      begin
    +          CentralHdr.Extra_Field_Length:=SizeOf(LocalZip64ExtHdr)+SizeOf(LocalZip64Fld);
    +      end else CentralHdr.Extra_Field_Length :=0;
    +
         FOutStream.Seek(0,soEnd);
         FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapCFH{$ENDIF}(CentralHdr),SizeOf(CentralHdr));
         FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
    +
         if IsZip64 then
           begin
           FOutStream.Seek(0,soEnd);
    @@ -1744,7 +1752,7 @@
     
         Inc(ACount);
         // Move past compressed file data to next header:
    -    if Iszip64 then
    +    if LocalZip64Fld.Compressed_Size > 0 then
           FOutStream.Seek(SavePos + LocalZip64Fld.Compressed_Size,soBeginning)
         else
           FOutStream.Seek(SavePos + LocalHdr.Compressed_Size,soBeginning);
    @@ -2093,7 +2101,7 @@
     Var
       Path: RawByteString;
       OldDirectorySeparators: set of char;
    -  
    +
     Begin
       { the default RTL behavior is broken on Unix platforms
         for Windows compatibility: it allows both '/' and '\'
    @@ -2521,7 +2529,10 @@
               end;
             end;
           // Move past extra fields and file comment to next header
    -      FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
    +      if File_Comment_Length > 0 then
    +          FZipStream.Seek(File_Comment_Length,soFromCurrent);
    +      // this doesn't work properly when zip file size is over 4Gb, so commented off
    +      //FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
           end;
         end;
     end;
    @@ -2605,7 +2616,7 @@
               OnProgress:=Self.OnProgress;
               OnPercent:=Self.OnPercent;
               DeCompress;
    -          Self.FTotPos := FTotPos; 
    +          Self.FTotPos := FTotPos;
               if Item.CRC32 <> Crc32Val then
                 raise EZipError.CreateFmt(SErrInvalidCRC,[Item.ArchiveFileName]);
             Finally
    
    7gb_zipper_v3.patch (3,162 bytes)

Activities

Marģers

2018-10-23 11:37

reporter  

tezipper.pas (2,037 bytes)
{ https://bugs.freepascal.org/view.php?id=34281 }
program tezipper;  {test TZipper class}

{$mode objfpc}
uses zipper, classes, sysutils;

procedure filldata(data : pbyte; size : dword);
var iA : dword;
    selfData : pbyte;
begin
     selfData := pbyte(@fillData); {code}
     for iA:=0 to size-1 do
     begin

          data^:=selfData^;
          inc(data);
          inc(selfData);
     end;
end;

procedure createFileWithSize(fname:string;size : qword);
var
    totalSize : qword;
    tmpData : pbyte;
    iA : dword;
    bufSize,wrBufSize, wrCount : dword;
    fn : file;


begin
     totalSize:=0;
     bufSize:=2048;
     wrBufSize:=bufSize;
     getMem(tmpData, bufSize);

     assign(fn, fname);
     rewrite(fn, 1);

     fillData ( tmpData, bufSize);
     while totalSize < size do
     begin
          if totalSize+wrBufSize> size then
               wrBufSize:=size-totalSize;
          blockWrite(fn,tmpData^, wrBufSize, wrCount);
          totalSize:=totalSize+wrCount;
          if wrCount < wrBufSize then break;
     end;

     close(fn);

     freeMem(tmpData, bufSize);
     writeln('File ',fname,' created. Size ',totalSize,'  (',totalSize / 1024 / 1024 / 1024:5:4,' Gb)');
end;

procedure zip(zipName, fname : string);
var
  OurZipper: TZipper;
begin

  OurZipper := TZipper.Create;
  try
    OurZipper.FileName := zipName;

     OurZipper.Entries.AddFileEntry(fname, fname);
     OurZipper.ZipAllFiles;
  finally
    OurZipper.Free;
  end;


end;

procedure unzip(zipName : string);
var
  UnZipper: TUnZipper;
begin
  UnZipper := TUnZipper.Create;
  try
    UnZipper.FileName := zipName;
    UnZipper.OutputPath := 'temp';
    UnZipper.Examine;
    UnZipper.UnZipAllFiles;
  finally
    UnZipper.Free;
  end;
end;

begin
	writeln;
     writeln('Create mega size file');
     createFileWithSize('mega7gb.tmp',4 * 1024*1024*1024+1024);
	write('Wait! Zipping.. ');
	zip ('zz.zip', 'mega7gb.tmp');
     writeln;

     write('Wait! UnZipping..');
	unzip('zz.zip');
     writeln;

	writeln('Zip test finished.');
end.
tezipper.pas (2,037 bytes)

Marģers

2018-10-23 11:42

reporter   ~0111520

possible solution

zipper.pp

line 1602
- if FZip.Size >= $FFFFFFFF then
+ if (LocalZip64Fld.Original_Size > 0) or (FZip.Size >= $FFFFFFFF) then

Marģers

2018-10-23 11:49

reporter   ~0111521

tezipper.pas will create 3 files:
mega7gb.tmp (size 4 Gb)
temp/mega7gb.tmp (size 4 Gb)
zz.zip

delete manually to save disk space

Thaddy de Koning

2018-10-23 13:32

reporter   ~0111522

I believe this is already fixed in 3.2.0

Marģers

2018-10-23 13:45

reporter   ~0111523

@Thaddy de Koning in current trunk it's not fixed yet

Marģers

2018-10-24 08:48

reporter  

zipper_7gb.patch (3,077 bytes)
Index: zipper.pp
===================================================================
--- zipper.pp	(revision 40017)
+++ zipper.pp	(working copy)
@@ -1609,6 +1609,8 @@
       begin
         Compressed_Size := FZip.Size;
         LocalZip64Fld.Compressed_Size := 0;
+        if LocalZip64Fld.Original_Size > 0 then
+          IsZip64 := true;
       end;
       if AZipVersionReqd > Extract_Version_Reqd then
         Extract_Version_Reqd := AZipVersionReqd;
@@ -1623,7 +1625,6 @@
   FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
   if IsZip64 then
   begin
-    LocalZip64ExtHdr.Header_ID:=ZIP64_HEADER_ID;
     FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapEDFH{$ENDIF}(LocalZip64ExtHdr),SizeOf(LocalZip64ExtHdr));
     FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapZ64EIF{$ENDIF}(LocalZip64Fld),SizeOf(LocalZip64Fld));
   end;
@@ -1682,7 +1683,7 @@
             end;
           end;
       // Move past extra fields
-      FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
+      //FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
       end;
     SavePos := FOutStream.Position;
     FillChar(CentralHdr,SizeOf(CentralHdr),0);
@@ -1732,9 +1733,16 @@
       else
         Local_Header_Offset := HdrPos;
       end;
+
+      if IsZip64 then
+      begin
+          Extra_Field_Length:=SizeOf(LocalZip64ExtHdr)+SizeOf(LocalZip64Fld);
+      end else Extra_Field_Length :=0;
+
     FOutStream.Seek(0,soEnd);
     FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapCFH{$ENDIF}(CentralHdr),SizeOf(CentralHdr));
     FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
+
     if IsZip64 then
       begin
       FOutStream.Seek(0,soEnd);
@@ -1744,7 +1752,7 @@
 
     Inc(ACount);
     // Move past compressed file data to next header:
-    if Iszip64 then
+    if LocalZip64Fld.Compressed_Size > 0 then
       FOutStream.Seek(SavePos + LocalZip64Fld.Compressed_Size,soBeginning)
     else
       FOutStream.Seek(SavePos + LocalHdr.Compressed_Size,soBeginning);
@@ -2093,7 +2101,7 @@
 Var
   Path: RawByteString;
   OldDirectorySeparators: set of char;
-  
+
 Begin
   { the default RTL behavior is broken on Unix platforms
     for Windows compatibility: it allows both '/' and '\'
@@ -2521,7 +2529,10 @@
           end;
         end;
       // Move past extra fields and file comment to next header
-      FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
+      if File_Comment_Length > 0 then
+          FZipStream.Seek(File_Comment_Length,soFromCurrent);
+      // this doesn't work properly when zip file size is over 4Gb, so commented off
+      //FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
       end;
     end;
 end;
@@ -2605,7 +2616,7 @@
           OnProgress:=Self.OnProgress;
           OnPercent:=Self.OnPercent;
           DeCompress;
-          Self.FTotPos := FTotPos; 
+          Self.FTotPos := FTotPos;
           if Item.CRC32 <> Crc32Val then
             raise EZipError.CreateFmt(SErrInvalidCRC,[Item.ArchiveFileName]);
         Finally
zipper_7gb.patch (3,077 bytes)

Marģers

2018-10-24 09:01

reporter   ~0111534

Last edited: 2018-10-24 09:39

View 4 revisions

zipper_7gb.patch solves two problems
1. file size is over 4Gb, but compressed size less than 4Gb.
2. archive itself is larger than 4Gb, but built of files smaller than 4Gb.

Do-wan Kim

2018-10-27 04:36

reporter  

7gb-zipper.pp.patch (3,170 bytes)
Index: packages/paszlib/src/zipper.pp
===================================================================
--- packages/paszlib/src/zipper.pp	(revision 40029)
+++ packages/paszlib/src/zipper.pp	(working copy)
@@ -1609,6 +1609,8 @@
       begin
         Compressed_Size := FZip.Size;
         LocalZip64Fld.Compressed_Size := 0;
+        if LocalZip64Fld.Original_Size > 0 then
+          IsZip64 := true;
       end;
       if AZipVersionReqd > Extract_Version_Reqd then
         Extract_Version_Reqd := AZipVersionReqd;
@@ -1623,7 +1625,6 @@
   FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
   if IsZip64 then
   begin
-    LocalZip64ExtHdr.Header_ID:=ZIP64_HEADER_ID;
     FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapEDFH{$ENDIF}(LocalZip64ExtHdr),SizeOf(LocalZip64ExtHdr));
     FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapZ64EIF{$ENDIF}(LocalZip64Fld),SizeOf(LocalZip64Fld));
   end;
@@ -1682,7 +1683,7 @@
             end;
           end;
       // Move past extra fields
-      FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
+      //FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
       end;
     SavePos := FOutStream.Position;
     FillChar(CentralHdr,SizeOf(CentralHdr),0);
@@ -1732,9 +1733,16 @@
       else
         Local_Header_Offset := HdrPos;
       end;
+
+      if IsZip64 then
+      begin
+          LocalHdr.Extra_Field_Length:=SizeOf(LocalZip64ExtHdr)+SizeOf(LocalZip64Fld);
+      end else LocalHdr.Extra_Field_Length :=0;
+
     FOutStream.Seek(0,soEnd);
     FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapCFH{$ENDIF}(CentralHdr),SizeOf(CentralHdr));
     FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
+
     if IsZip64 then
       begin
       FOutStream.Seek(0,soEnd);
@@ -1744,7 +1752,7 @@
 
     Inc(ACount);
     // Move past compressed file data to next header:
-    if Iszip64 then
+    if LocalZip64Fld.Compressed_Size > 0 then
       FOutStream.Seek(SavePos + LocalZip64Fld.Compressed_Size,soBeginning)
     else
       FOutStream.Seek(SavePos + LocalHdr.Compressed_Size,soBeginning);
@@ -2093,7 +2101,7 @@
 Var
   Path: RawByteString;
   OldDirectorySeparators: set of char;
-  
+
 Begin
   { the default RTL behavior is broken on Unix platforms
     for Windows compatibility: it allows both '/' and '\'
@@ -2521,7 +2529,10 @@
           end;
         end;
       // Move past extra fields and file comment to next header
-      FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
+      if File_Comment_Length > 0 then
+          FZipStream.Seek(File_Comment_Length,soFromCurrent);
+      // this doesn't work properly when zip file size is over 4Gb, so commented off
+      //FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
       end;
     end;
 end;
@@ -2605,7 +2616,7 @@
           OnProgress:=Self.OnProgress;
           OnPercent:=Self.OnPercent;
           DeCompress;
-          Self.FTotPos := FTotPos; 
+          Self.FTotPos := FTotPos;
           if Item.CRC32 <> Crc32Val then
             raise EZipError.CreateFmt(SErrInvalidCRC,[Item.ArchiveFileName]);
         Finally
7gb-zipper.pp.patch (3,170 bytes)

Marģers

2018-10-28 09:02

reporter  

7gb_zipper_v3.patch (3,162 bytes)
Index: packages/paszlib/src/zipper.pp
===================================================================
--- packages/paszlib/src/zipper.pp	(revision 40057)
+++ packages/paszlib/src/zipper.pp	(working copy)
@@ -1609,6 +1609,8 @@
       begin
         Compressed_Size := FZip.Size;
         LocalZip64Fld.Compressed_Size := 0;
+        if LocalZip64Fld.Original_Size > 0 then
+          IsZip64 := true;
       end;
       if AZipVersionReqd > Extract_Version_Reqd then
         Extract_Version_Reqd := AZipVersionReqd;
@@ -1623,7 +1625,6 @@
   FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
   if IsZip64 then
   begin
-    LocalZip64ExtHdr.Header_ID:=ZIP64_HEADER_ID;
     FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapEDFH{$ENDIF}(LocalZip64ExtHdr),SizeOf(LocalZip64ExtHdr));
     FOutStream.WriteBuffer({$IFDEF ENDIAN_BIG}SwapZ64EIF{$ENDIF}(LocalZip64Fld),SizeOf(LocalZip64Fld));
   end;
@@ -1682,7 +1683,7 @@
             end;
           end;
       // Move past extra fields
-      FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
+      //FOutStream.Seek(SavePos+LocalHdr.Extra_Field_Length,soFromBeginning);
       end;
     SavePos := FOutStream.Position;
     FillChar(CentralHdr,SizeOf(CentralHdr),0);
@@ -1732,9 +1733,16 @@
       else
         Local_Header_Offset := HdrPos;
       end;
+
+      if IsZip64 then
+      begin
+          CentralHdr.Extra_Field_Length:=SizeOf(LocalZip64ExtHdr)+SizeOf(LocalZip64Fld);
+      end else CentralHdr.Extra_Field_Length :=0;
+
     FOutStream.Seek(0,soEnd);
     FOutStream.WriteBuffer({$IFDEF FPC_BIG_ENDIAN}SwapCFH{$ENDIF}(CentralHdr),SizeOf(CentralHdr));
     FOutStream.WriteBuffer(ZFileName[1],Length(ZFileName));
+
     if IsZip64 then
       begin
       FOutStream.Seek(0,soEnd);
@@ -1744,7 +1752,7 @@
 
     Inc(ACount);
     // Move past compressed file data to next header:
-    if Iszip64 then
+    if LocalZip64Fld.Compressed_Size > 0 then
       FOutStream.Seek(SavePos + LocalZip64Fld.Compressed_Size,soBeginning)
     else
       FOutStream.Seek(SavePos + LocalHdr.Compressed_Size,soBeginning);
@@ -2093,7 +2101,7 @@
 Var
   Path: RawByteString;
   OldDirectorySeparators: set of char;
-  
+
 Begin
   { the default RTL behavior is broken on Unix platforms
     for Windows compatibility: it allows both '/' and '\'
@@ -2521,7 +2529,10 @@
           end;
         end;
       // Move past extra fields and file comment to next header
-      FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
+      if File_Comment_Length > 0 then
+          FZipStream.Seek(File_Comment_Length,soFromCurrent);
+      // this doesn't work properly when zip file size is over 4Gb, so commented off
+      //FZipStream.Seek(SavePos+Extra_Field_Length+File_Comment_Length,soFromBeginning);
       end;
     end;
 end;
@@ -2605,7 +2616,7 @@
           OnProgress:=Self.OnProgress;
           OnPercent:=Self.OnPercent;
           DeCompress;
-          Self.FTotPos := FTotPos; 
+          Self.FTotPos := FTotPos;
           if Item.CRC32 <> Crc32Val then
             raise EZipError.CreateFmt(SErrInvalidCRC,[Item.ArchiveFileName]);
         Finally
7gb_zipper_v3.patch (3,162 bytes)

Marģers

2018-10-28 09:06

reporter   ~0111630

added 7gb_zipper_v3.patch where fixed missing "CentralHdr."

Ross

2018-11-03 20:52

reporter   ~0111762

I applied changes in 7gb_zipper_v3.patch and everything now works as expected.
Nice work guys, thanks.

Michael Van Canneyt

2018-11-04 13:04

administrator   ~0111781

Tested and applied patch. Thank you very much for the patch!

A good exercise for the speed of SSD disks :)

Issue History

Date Modified Username Field Change
2018-09-14 20:21 Ross New Issue
2018-10-23 11:37 Marģers File Added: tezipper.pas
2018-10-23 11:42 Marģers Note Added: 0111520
2018-10-23 11:49 Marģers Note Added: 0111521
2018-10-23 13:32 Thaddy de Koning Note Added: 0111522
2018-10-23 13:45 Marģers Note Added: 0111523
2018-10-24 08:48 Marģers File Added: zipper_7gb.patch
2018-10-24 09:01 Marģers Note Added: 0111534
2018-10-24 09:02 Marģers Note Edited: 0111534 View Revisions
2018-10-24 09:03 Marģers Note Edited: 0111534 View Revisions
2018-10-24 09:39 Marģers Note Edited: 0111534 View Revisions
2018-10-26 10:34 Michael Van Canneyt Assigned To => Michael Van Canneyt
2018-10-26 10:34 Michael Van Canneyt Status new => assigned
2018-10-27 04:36 Do-wan Kim File Added: 7gb-zipper.pp.patch
2018-10-28 09:02 Marģers File Added: 7gb_zipper_v3.patch
2018-10-28 09:06 Marģers Note Added: 0111630
2018-11-03 20:52 Ross Note Added: 0111762
2018-11-04 13:04 Michael Van Canneyt Fixed in Revision => 40215
2018-11-04 13:04 Michael Van Canneyt Note Added: 0111781
2018-11-04 13:04 Michael Van Canneyt Status assigned => resolved
2018-11-04 13:04 Michael Van Canneyt Fixed in Version => 3.3.1
2018-11-04 13:04 Michael Van Canneyt Resolution open => fixed
2018-11-04 13:04 Michael Van Canneyt Target Version => 3.2.0