src/charsets.c

   1 /*
   2   Copyright 1998-2003 Victor Wagner
   3   Copyright 2003 Alex Ott
   4   This file is released under the GPL.  Details can be
   5   found in the file COPYING accompanying this distribution.
   6 */
   7 #ifdef HAVE_CONFIG_H
   8 #include <config.h>
   9 #endif
  10 #include <stdio.h>
  11 #include <string.h>
  12 #include <stdlib.h>
  13 #include "catdoc.h"
  14
  15 char *charset_path=CHARSETPATH;
  16 char *source_csname=SOURCE_CHARSET, *dest_csname=TARGET_CHARSET;
  17 short int * source_charset;
  18 int unknown_as_hex=0;
  19 char bad_char[]=UNKNOWN_CHAR;
  20 CHARSET target_charset;
  21 /************************************************************************/
  22 /* Converts char in input charset into unicode representation           */
  23 /* Should be converted to macro                                         */
  24 /************************************************************************/
  25 int to_unicode (short int *charset, int c) {
  26         return charset[c];
  27 }
  28 /************************************************************************/
  29 /* Search inverse charset record for given unicode char and returns     */
  30 /* 0-255 char value if found, -1 otherwise                              */
  31 /************************************************************************/
  32 int from_unicode (CHARSET charset, int u) {
  33         short int *p;
  34         /* This is really assignment, not comparation */
  35         if ((p=charset[(unsigned)u>>8])) {
  36                 return p[u & 0xff];
  37         } else {
  38                 return -1;
  39         }
  40 }
  41 /************************************************************************/
  42 /*  Converts direct (charset -> unicode) to reverse map                 */
  43 /************************************************************************/
  44 CHARSET make_reverse_map(short int *charset) {
  45         CHARSET newmap=calloc(sizeof(short int *), 256);
  46         int i,j,k,l;
  47         short int *p;
  48         if (! charset) {
  49                 return NULL;
  50         }
  51         for (i=0;i<256;i++) {
  52                 k= charset[i];
  53                 j=  (unsigned)k>>8;
  54                 if (!newmap[j]) {
  55                         newmap[j] = malloc(sizeof(short int *)*256);
  56                         if (!newmap[j]) {
  57                                 fprintf(stderr,"Insufficient memory for  charset\n");
  58                                 exit(1);
  59                         }
  60                         for (l=0,p=newmap[j];l<256;l++,p++) *p=-1;
  61                 }
  62                 p=newmap[j];
  63                 p[k & 0xff]=i;
  64         }
  65         return newmap;
  66 }
  67
  68 /************************************************************************/
  69 /* Reads charset file (as got from ftp.unicode.org) and returns array of*/
  70 /* 256 short ints (malloced) mapping from charset t unicode             */
  71 /************************************************************************/
  72 short int * read_charset(const char *filename) {
  73         char *path;
  74         FILE *f;
  75         short int *new=calloc(sizeof(short int),256);
  76         int c;
  77         long int uc;
  78         path= find_file(stradd(filename,CHARSET_EXT),charset_path);
  79         if (!path) {
  80                 fprintf(stderr,"Cannot load charset %s - file not found\n",filename);
  81                 return NULL;
  82         }
  83         f=fopen(path,"rb");
  84
  85         if (!f) {
  86                 perror(path);
  87                 return NULL;
  88         }
  89         if (input_buffer)
  90                 setvbuf(f,input_buffer,_IOFBF,FILE_BUFFER);
  91         /* defaults */
  92         for (c=0;c<32;c++) {
  93                 new[c]=c;
  94         }
  95         while (!feof(f)) {
  96                 if (fscanf(f,"%i %li",&c,&uc)==2) {
  97                         if (c<0||c>255||uc<0||(uc>0xFEFE&& uc!=0xFFFE)) {
  98                                 fprintf(stderr,"Invalid charset file %s\n",path);
  99                                 fclose(f);
 100                                 return NULL;
 101                         }
 102                         new[c]=uc;
 103                 }
 104                 while((fgetc(f)!='\n')&&!feof(f)) ;
 105         }
 106         fclose (f);
 107         free(path);
 108         return new;
 109 }
 110
 111
 112 /************************************************************************/
 113 /* Reads 8-bit char and convers it from source charset                  */
 114 /************************************************************************/
 115
 116 int get_8bit_char (FILE *f,long *offset,long fileend)
 117 {
 118         unsigned char buf;
 119         if (catdoc_read(&buf, 1, 1, f)==0) return EOF;
 120         (*offset)++;
 121         return to_unicode(source_charset,buf);
 122 }
 123
 124
 125 /************************************************************************/
 126 /* Reads 16-bit unicode value. MS-Word runs on LSB-first machine only,  */
 127 /* so read lsb first always and don't care about proper bit order       */
 128 /************************************************************************/
 129
 130 int get_utf16lsb (FILE *f,long *offset,long fileend) {
 131         unsigned char buf[2];
 132     int result;
 133         result=catdoc_read(buf, 1, 2, f);
 134         if (result<0) {
 135                 perror("read:");
 136                 exit(1);
 137         }
 138         if (result !=2) {
 139                 return EOF;
 140         }
 141         (*offset)+=2;
 142         return ((int)buf[1])|(((int)buf[0])<<8);
 143 }
 144
 145 /************************************************************************/
 146 /* Reads 16-bit unicode value written in MSB order. For processing
 147  * non-word files            .                                          */
 148 /************************************************************************/
 149 int get_utf16msb (FILE *f,long *offset,long fileend) {
 150         unsigned char buf[2];
 151     int result;
 152         result=catdoc_read(buf, 1, 2, f);
 153         if (result<0) {
 154                 perror("read:");
 155                 exit(1);
 156         }
 157         if (result !=2) {
 158                 return EOF;
 159         }
 160         (*offset)+=2;
 161         return ((int)buf[0])|(((int)buf[1])<<8);
 162 }
 163
 164 int get_utf8 (FILE *f,long *offset,long fileend) {
 165         unsigned char buf[3];
 166         int c;
 167     int result;
 168         result=catdoc_read(buf, 1, 1, f);
 169         if (result<0) {
 170                 perror("read");
 171                 exit(1);
 172         }
 173         if (result==0) return EOF;
 174         c=buf[0];
 175         if (c<0x80)
 176                 return c;
 177         if (c <0xC0)
 178                 return 0xfeff; /*skip corrupted sequebces*/
 179         if (c <0xE0) {
 180                 if (catdoc_read(buf+1, 1, 1, f)<=0) return EOF;
 181                 return ((c & 0x1F)<<6 | ((char)buf[1] & 0x3F));
 182         }
 183         if (c <0xF0) {
 184                 if (catdoc_read(buf+1, 1, 2, f)<=2) return (int)EOF;
 185                 return ((c & 0x0F)<<12)|
 186                         ((buf[1] & 0x3f)<<6)|
 187                                          (buf[2] & 0x3f);
 188         }
 189         return 0xFEFF;
 190 }
 191
 192 /**************************************************************************/
 193 /*  Converts unicode char to output charset sequence. Coversion have      */
 194 /*  three steps: 1. Replacement map is searched for the character in case */
 195 /* it is not allowed for output format (% in TeX, < in HTML               */
 196 /* 2. target charset is searched for this unicode char, if it wasn't      */
 197 /*  replaced. If not found, then 3. Substitution map is searched          */
 198 /**************************************************************************/
 199 char *convert_char(int uc) {
 200         static char plain_char[]="a"; /*placeholder for one-char sequences */
 201         static char hexbuf[8];
 202         char *mapped;
 203         int c;
 204         if ((mapped=map_subst(spec_chars,uc))) return mapped;
 205         if (target_charset) {
 206                 c =from_unicode(target_charset,uc);
 207                 if (c>=0) {
 208                         *plain_char=c;
 209                         return plain_char;
 210                 }
 211                 if ((mapped = map_subst(replacements,uc))) return mapped;
 212                 if (unknown_as_hex) {
 213                         sprintf(hexbuf,"\\x%04X",(unsigned)uc);
 214                         /* This sprintf is safe, becouse uc is unicode character code,
 215                            which cannot be greater than 0xFFFE. It is ensured by routines
 216                            in reader.c
 217                            */
 218                         return hexbuf;
 219                 }
 220                 return  bad_char;
 221         } else {
 222                 /* NULL target charset means UTF-8 output */
 223                 return to_utf8(uc);
 224         }
 225 }
 226 /******************************************************************/
 227 /* Converts given unicode character to the utf-8 sequence         */
 228 /* in the static string buffer. Buffer wouldbe overwritten upon   */
 229 /* next call                                                      */
 230 /******************************************************************/
 231 char *to_utf8(unsigned int uc) {
 232         static char utfbuffer[4]; /* it shouldn't overflow becouse we never deal
 233                                                                  with chars greater than 65535*/
 234         int count=0;
 235         if (uc< 0x80) {
 236                 utfbuffer[0]=uc;
 237                 count=1;
 238         } else  {
 239                 if (uc < 0x800) {
 240                         utfbuffer[count++]=0xC0 | (uc >> 6);
 241                 } else {
 242                         utfbuffer[count++]=0xE0 | (uc >>12);
 243                         utfbuffer[count++]=0x80 | ((uc >>6) &0x3F);
 244                 }
 245                 utfbuffer[count++]=0x80 | (uc & 0x3F);
 246         }
 247         utfbuffer[count]=0;
 248         return utfbuffer;
 249 }
 250
 251 struct cp_map {
 252         int codepage;
 253         char *charset_name;
 254 };
 255
 256 struct cp_map cp_to_charset [] = {
 257         {10000,"mac-roman"},
 258         {10001,"mac-japanese"},
 259         {10002,"mac-tchinese"},
 260         {10003,"mac-korean"},
 261         {10004,"mac-arabic"},
 262         {10005,"mac-hebrew"},
 263         {10006,"mac-greek1"},
 264         {10007,"mac-cyrillic"},
 265         {10008,"mac-schinese"},
 266         {10010,"mac-romania"},
 267         {10017,"mac-ukraine"},
 268         {10021,"mac-thai"},
 269         {10029,"mac-centeuro"},
 270         {10079,"mac-iselandic"},
 271         {10081,"mac-turkish"},
 272         {10082,"mac-croatia"},
 273         {20866,"koi8-r"},
 274         {28591,"8859-1"},
 275         {28592,"8859-2"},
 276         {28593,"8859-3"},
 277         {28594,"8859-4"},
 278         {28595,"8859-5"},
 279         {28596,"8859-6"},
 280         {28597,"8859-7"},
 281         {28598,"8859-8"},
 282         {28599,"8859-9"},
 283         {28605,"8859-15"},
 284         {65001,"utf-8"},
 285     {0,NULL}};
 286 const char *charset_from_codepage(unsigned int codepage) {
 287
 288         static char buffer[7];
 289         struct cp_map *cp;
 290         if (codepage==1200||codepage==1201) {
 291                 /* For UCS2 */
 292                 return "";
 293         } else
 294         if (codepage<10000) {
 295                 sprintf(buffer,"cp%d",codepage);
 296                 return buffer;
 297         } else {
 298                 for (cp = cp_to_charset;cp->codepage!=0&& cp->codepage!=codepage;cp++);
 299                 return cp->charset_name;
 300         }
 301 }