View Issue Details

IDProjectCategoryView StatusLast Update
0020531FPCPackagespublic2015-01-21 13:10
Reporterbarlone Assigned To 
PrioritynormalSeveritytweakReproducibilityN/A
Status newResolutionopen 
OSWindows XP 
Product Version2.5.1 
Summary0020531: Package iconvenc fixed and tested for i386-win32 target
DescriptionI adapt package `iconvenc` to i386-win32 targer via MinGW port of libiconenc library. Tested on Win32 and Linux32. Also fixed error with buffer trimming and make some refactoring. I cant test package on Win64, but 64bit version of libiconenc available (on http://sourceforge.net/projects/mingw-w64/).
TagsNo tags attached.
Fixed in Revision26008
FPCOldBugId
FPCTarget
Attached Files

Relationships

related to 0025297 closedMarco van de Voort iconvert null ponint error 

Activities

2011-10-21 13:07

 

iconvenc.pas.patch (2,617 bytes)   
Index: iconvenc.pas
===================================================================
--- iconvenc.pas	(revision 19485)
+++ iconvenc.pas	(working copy)
@@ -21,8 +21,17 @@
 
 
 uses
-  ctypes,unixtype,baseunix,
-  initc;
+  ctypes
+{$IFDEF UNIX}
+  ,unixtype,baseunix,initc
+{$ELSE}
+ {$IFDEF WINDOWS}
+   {$MESSAGE NOTE This package use MinGW port of libiconvenc for Windows target}
+ {$ELSE} 
+   {$MESSAGE ERROR Target not supported'}
+ {$ENDIF}
+{$ENDIF}
+  ;
 
 const
   n = 1;
@@ -34,11 +43,15 @@
 type
    piconv_t = ^iconv_t;
    iconv_t = pointer;
+{$IFDEF WINDOWS}
+   psize_t = pcsize_t;
+{$ENDIF}
 
    Ticonv_open = function(__tocode: pchar; __fromcode: pchar): iconv_t; cdecl;
    Ticonv = function(__cd: iconv_t; __inbuf: ppchar; __inbytesleft: psize_t; __outbuf: ppchar; __outbytesleft: psize_t): size_t; cdecl;
    Ticonv_close = function(__cd: iconv_t): cint; cdecl;
 
+{$IFDEF UNIX}
 {$if not defined(linux) and not defined(solaris)}  // Linux (and maybe glibc pl
    {$if defined(haiku)}
     {$linklib textencoding}
@@ -47,23 +60,29 @@
    {$endif}
  {$define useiconv}
 {$endif linux}
+{$ENDIF}
           
 Const
-{$ifndef useiconv}
-  libiconvname='c';  // is in libc under Linux.
-{$else}
-  {$ifdef haiku}
-    libiconvname='textencoding';  // is in libtextencoding under Haiku
+{$IFDEF UNIX}
+  {$ifndef useiconv}
+    libiconvname='c';  // is in libc under Linux.
   {$else}
-    libiconvname='iconv';
+    {$ifdef haiku}
+      libiconvname='textencoding';  // is in libtextencoding under Haiku
+    {$else}
+      libiconvname='iconv';
+    {$endif}
   {$endif}
-{$endif}
 
-{$if (defined(darwin) and defined(cpupowerpc32)) or defined(haiku)}
-  iconvprefix='lib';  
-{$else}
-  iconvprefix='';
-{$endif}
+  {$if (defined(darwin) and defined(cpupowerpc32)) or defined(haiku)}
+    iconvprefix='lib';  
+  {$else}
+    iconvprefix='';
+  {$endif}
+{$ELSE}
+    libiconvname='libiconv-2.dll';
+    iconvprefix='lib';  
+{$ENDIF}
                            
 function iconv_open(__tocode: pchar; __fromcode: pchar): iconv_t; cdecl; external libiconvname name iconvprefix+'iconv_open';
 function iconv (__cd: iconv_t; __inbuf: ppchar; __inbytesleft: psize_t; __outbuf: ppchar; __outbytesleft: psize_t): size_t; cdecl; external libiconvname name iconvprefix+'iconv';
@@ -72,7 +91,7 @@
 var
   IconvLibFound: boolean = False;
 
-function Iconvert(s: string; var res: string; FromEncoding, ToEncoding: string): cint;
+function Iconvert(const s: string; var res: string; const FromEncoding, ToEncoding: string): cint;
 function InitIconv(var error: string): boolean;
 
 implementation
iconvenc.pas.patch (2,617 bytes)   

2011-10-21 13:07

 

iconvenc_dyn.pas.patch (3,628 bytes)   
Index: iconvenc_dyn.pas
===================================================================
--- iconvenc_dyn.pas	(revision 19485)
+++ iconvenc_dyn.pas	(working copy)
@@ -20,55 +20,84 @@
 {$mode objfpc}{$H+}
 
 uses
-  ctypes,unixtype,baseunix,
+  ctypes,
+{$IFDEF UNIX}
+  unixtype,baseunix,
   dl,
   initc;
+{$ELSE}
+ {$IFDEF WINDOWS}
+   sysutils, 
+   windows;
+ {$ELSE} 
+   {$MESSAGE ERROR Target not supported'}
+ {$ENDIF}
+{$ENDIF}
 
+{$ifdef beos}
 const
-  n = 1;
-
-{$ifdef beos}
   ESysEILSEQ = EILSEQ;
 {$endif}
 
 type
    piconv_t = ^iconv_t;
    iconv_t = pointer;
+{$IFDEF WINDOWS}
+   psize_t = pcsize_t;
+{$ENDIF}
 
    Ticonv_open = function(__tocode: pchar; __fromcode: pchar): iconv_t; cdecl;
    Ticonv = function(__cd: iconv_t; __inbuf: ppchar; __inbytesleft: psize_t; __outbuf: ppchar; __outbytesleft: psize_t): size_t; cdecl;
    Ticonv_close = function(__cd: iconv_t): cint; cdecl;
 
 var
+{$IFDEF UNIX}
   iconv_lib: pointer;
+{$ELSE}
+  iconv_lib: THandle;
+{$ENDIF}
   iconv_open: Ticonv_open;
   iconv: Ticonv;
   iconv_close: Ticonv_close;
   IconvLibFound: boolean = False;
 
-function TryLoadLib(LibName: string; var error: string): boolean; // can be used to load non standard libname
-function Iconvert(s: string; var res: string; FromEncoding, ToEncoding: string): cint;
+function TryLoadLib(const LibName: string; var error: string): boolean; // can be used to load non standard libname
+function Iconvert(const s: string; var res: string; const FromEncoding, ToEncoding: string): cint;
 function InitIconv(var error: string): boolean;
 
 implementation
 
-function TryLoadLib(LibName: string; var error: string): boolean;
+function TryLoadLib(const LibName: string; var error: string): boolean;
 
-    function resolvesymbol (var funcptr; symbol: string): Boolean;
+{$IFDEF WINDOWS}
+    function dlerror: string;
     begin
+      Result := SysErrorMessage(GetLastError());
+    end;
+{$ENDIF}     
+
+    function resolvesymbol (var funcptr; const symbol: string): Boolean;
+    begin
+{$IFDEF UNIX}
       pointer(funcptr) := pointer(dlsym(iconv_lib, pchar(symbol)));
+{$ELSE}
+      pointer(funcptr) := pointer(GetProcAddress(iconv_lib, pchar(symbol)));
+{$ENDIF}
       result := assigned(pointer(funcptr));
       if not result then
-        error := error+#13#10+dlerror();
+        error := error+LineEnding+dlerror();
     end;
 
-var
-  res: boolean;
 begin
   result := false;
-  Error := Error+#13#10'Trying '+LibName;
+  Error := Error+LineEnding+'Trying '+LibName;
+{$IFDEF UNIX}
   iconv_lib := dlopen(pchar(libname), RTLD_NOW);
   if Assigned(iconv_lib) then
+{$ELSE}
+  iconv_lib := LoadLibrary(pchar(libname));
+  if iconv_lib <> 0 then
+{$ENDIF}
   begin
     result := true;
     result := result and resolvesymbol(pointer(iconv),'iconv');
@@ -81,10 +110,8 @@
         result := result and resolvesymbol(pointer(iconv_open),'libiconv_open');
         result := result and resolvesymbol(pointer(iconv_close),'libiconv_close');
       end;
-//    if not res then
-//      dlclose(iconv_lib);
   end else
-    error:=error+#13#10+dlerror();
+    error:=error+LineEnding+dlerror();
 end;
 
 
@@ -92,11 +119,16 @@
 begin
   result := true;
   error := '';
+{$IFDEF UNIX}
   if not TryLoadLib('libc.so.6', error) then
     if not TryLoadLib('libiconv.so', error) then
     {$if defined(haiku)}
         if not TryLoadLib('libtextencoding.so', error) then
       {$ifend}
+{$ELSE}
+  if not TryLoadLib('libiconv-2.dll', error) then
+    if not TryLoadLib('libiconv.dll', error) then
+{$ENDIF}
       result := false; 
   iconvlibfound := iconvlibfound or result;
 end;
iconvenc_dyn.pas.patch (3,628 bytes)   

2011-10-21 13:07

 

iconvert.inc.patch (1,163 bytes)   
Index: iconvert.inc
===================================================================
--- iconvert.inc	(revision 19485)
+++ iconvert.inc	(working copy)
@@ -3,12 +3,14 @@
  {$define noerrnoiconv}
 {$endif}
 
-function Iconvert(S: string; var Res: string; FromEncoding, ToEncoding: string): cint;
+function Iconvert(const s: string; var res: string; const FromEncoding, ToEncoding: string): cint;
 var
   InLen, OutLen, Offset: size_t;
   Src, Dst: pchar;
   H: iconv_t;
+{$ifndef noerrnoiconv}
   lerr: cint;
+{$endif}
   iconvres: size_t;
 begin
   H := iconv_open(PChar(ToEncoding), PChar(FromEncoding));
@@ -94,11 +96,12 @@
    {$endif}
     // trim output buffer
     SetLength(Res, Length(Res) - Outlen);
-  finally Offset:=Dst-PChar(Res);
-        SetLength(Res, Length(Res)+InLen*2+5); // 5 is minimally one utf-8 char
-        Dst:=PChar(Res)+Offset;
-        OutLen:=Length(Res)-Offset;
-        iconv(H, nil, nil, @Dst, @Outlen);
+  finally 
+    Offset:=Dst-PChar(Res);
+    SetLength(Res, Length(Res)+InLen*2);
+    Dst:=PChar(Res)+Offset;
+    OutLen:=Length(Res)-Offset;
+    iconv(H, nil, nil, @Dst, @Outlen);
 
     iconv_close(H);
   end;
iconvert.inc.patch (1,163 bytes)   

2011-10-21 13:07

 

iconvtest.pp.patch (1,224 bytes)   
Index: iconvtest.pp
===================================================================
--- iconvtest.pp	(revision 19485)
+++ iconvtest.pp	(working copy)
@@ -18,17 +18,34 @@
 
 {$mode objfpc}{$H+}
 
+{$DEFINE LOADDYNAMIC}
+{$DEFINE TEST_CP1251}
+{.$DEFINE TEST_CP1255}
+
 uses
   SysUtils,
-  iconvenc;
+{$IFDEF LOADDYNAMIC}
+  iconvenc_dyn
+{$ELSE}
+  iconvenc
+{$ENDIF}
+  ;
 
+
+{$IFDEF TEST_CP1255}
 // some random Hebrew string for testing in CP1255
 Const InputString : array[0..21] of char = 
           (#$e0,#$e1,#$e2,#$e3,#$e4,#$e5,#$e6,#$e7,
 	   #$e8,#$e9,#$eb,#$ec,#$ee,#$f0,#$f1,#$f2,
   	   #$f4,#$f6,#$f7,#$f8,#$f9,#$fa);
       InputEncoding = 'CP1255';
-   
+{$ENDIF}
+
+{$IFDEF TEST_CP1251}
+Const InputString : string = '� ���� ��� ������, �� � ����';
+      InputEncoding = 'CP1251';
+{$ENDIF}   
+
 procedure DoOneConversion(TargetEncoding:string);
 var
   fn,res: string;
@@ -56,14 +73,13 @@
 
 var s : string;
 begin
-  {$IFDEF LOADDYNAMIC}
   if not InitIconv(s) then
     begin
       Writeln('Iconv initialization failed:',s);
       halt;
     end ;
- {$ENDIF}
- DoOneConversion('UTF-8');
- DoOneConversion('UTF-16');
+
+  DoOneConversion('UTF-8');
+  DoOneConversion('UTF-16');
 end.
 
iconvtest.pp.patch (1,224 bytes)   

2011-10-21 13:08

 

Makefile.fpc.patch (988 bytes)   
Index: Makefile.fpc
===================================================================
--- Makefile.fpc	(revision 19485)
+++ Makefile.fpc	(working copy)
@@ -52,7 +52,7 @@
 dirs_win32=fv winunits-base winunits-jedi fcl-web fastcgi ibase mysql zlib oracle odbc postgres sqlite imagemagick \
                gdbint libpng mad tcl opengl gtk1 gtk2 librsvg a52 cdrom fpgtk openal fftw lua fcl-extra zorba \
                oggvorbis sdl openssl graph pcap  httpd22 pxlib numlib winceunits cairo libxml \
-               gmp opencl libsee nvapi ptc aspell
+               gmp opencl libsee nvapi ptc aspell iconvenc
 dirs_win64=fv winunits-base winunits-jedi fcl-web fastcgi ibase mysql zlib oracle odbc postgres sqlite imagemagick gdbint \
                 tcl opengl gtk1 fpgtk fftw sdl openssl cdrom  httpd22 numlib fcl-extra opencl nvapi ptc graph
 dirs_wince=winceunits httpd22 fcl-web fastcgi tcl fftw unzip zlib sqlite mysql ibase postgres oracle odbc sdl openssl oggvorbis numlib
Makefile.fpc.patch (988 bytes)   

barlone

2011-11-11 12:39

reporter   ~0054013

If for any reason windows port is not interesting, i think it is still desirable to fix the problem with buffer trimming in Iconvert function (iconvert.inc.patch).

Marco van de Voort

2013-11-10 13:58

manager   ~0071244

Last edited: 2015-01-21 13:10

View 4 revisions

Back then I didn't commit that since I knew the +5 was added for a reason. But I didn't see immediately what _was_ wrong (or how to test), so I just left it.

Looking at it today after Jonas' hints, the problem is not the +5, but not trimming it afterwards. Fixing that (r26008) I also noticed that in the unix code, one iconv() call is not checked for result. That is IMHO a critical bug. (also fixed in r26008).

As to the windows port, I don't generally support or use mingw libs, and thus don't have an installation to test. Which is more important here because it is not a simple enabling for windows, but code base on a different principle. TBD.

Issue History

Date Modified Username Field Change
2011-10-21 13:07 barlone New Issue
2011-10-21 13:07 barlone File Added: iconvenc.pas.patch
2011-10-21 13:07 barlone File Added: iconvenc_dyn.pas.patch
2011-10-21 13:07 barlone File Added: iconvert.inc.patch
2011-10-21 13:07 barlone File Added: iconvtest.pp.patch
2011-10-21 13:08 barlone File Added: Makefile.fpc.patch
2011-11-11 12:39 barlone Note Added: 0054013
2013-11-09 14:20 Jonas Maebe Relationship added related to 0025297
2013-11-10 13:58 Marco van de Voort Note Added: 0071244
2013-11-10 13:59 Marco van de Voort Fixed in Revision => 26008
2013-11-10 14:15 Marco van de Voort Note Edited: 0071244 View Revisions
2014-01-08 15:31 Marco van de Voort Note Edited: 0071244 View Revisions
2015-01-21 13:10 Marco van de Voort Note Edited: 0071244 View Revisions