charset.c File Reference

#include "gwlib/gwlib.h"
#include <errno.h>
#include <iconv.h>
#include "gwlib/latin1_to_gsm.h"

Include dependency graph for charset.c:

Go to the source code of this file.

Data Structures
struct	alias_t
Defines
#define	NRP   '?'
Typedefs
typedef alias_t	alias_t
Functions
void	charset_init ()
void	charset_shutdown ()
void	charset_gsm_to_utf8 (Octstr *ostr)
void	charset_utf8_to_gsm (Octstr *ostr)
void	charset_gsm_to_latin1 (Octstr *ostr)
void	charset_latin1_to_gsm (Octstr *ostr)
void	charset_gsm_to_nrc_iso_21_german (Octstr *ostr)
void	charset_nrc_iso_21_german_to_gsm (Octstr *ostr)
int	charset_gsm_truncate (Octstr *gsm, long max)
int	charset_to_utf8 (Octstr *from, Octstr **to, Octstr *charset_from)
int	charset_from_utf8 (Octstr *utf8, Octstr **to, Octstr *charset_to)
int	charset_convert (Octstr *string, char *charset_from, char *charset_to)
Variables
struct {
int   gsmesc
int   latin1
}	gsm_esctolatin1 []
struct {
int   gsmesc
int   unichar
}	gsm_esctouni []
const unsigned char	gsm_to_latin1 [128]
const int	gsm_to_unicode [128]
alias_t	chars_aliases []

---

Define Documentation

#define NRP   '?'

Definition at line 73 of file charset.c.

---

Typedef Documentation

typedef struct alias_t alias_t

Definition at line 183 of file charset.c.

---

Function Documentation

int charset_convert (  Octstr *  string, char *  charset_from, char *  charset_to )

Definition at line 590 of file charset.c. References debug(), error(), octstr_append_data(), octstr_delete(), octstr_get_cstr, octstr_len(), and string. Referenced by charset_processing(), convert_addr_from_pdu(), data_sm_to_msg(), httpsmsc_send(), msg_to_pdu(), normalize_charset(), parse_attr_value(), parse_text(), pdu_to_msg(), soap_msgdata_attribute(), soap_msgdata_deps(), and soap_o2o_msgdata_attribute(). { #if HAVE_ICONV_H char *from_buf, *to_buf, *pointer; size_t inbytes, outbytes; int ret; iconv_t cd; if (!charset_from || !charset_to || !string) /* sanity check */ return -1; cd = iconv_open(charset_to, charset_from); /* Did I succeed in getting a conversion descriptor ? */ if (cd == (iconv_t)(-1)) { /* I guess not */ error(0,"Failed to convert string from <%s> to <%s> - probably broken type names.", charset_from, charset_to); return -1; } from_buf = octstr_get_cstr(string); /* allocate max sized buffer, assuming target encoding may be 4 byte unicode */ inbytes = octstr_len(string); outbytes = sizeof(char) * octstr_len(string) * 4; pointer = to_buf = gw_malloc(outbytes + 1); memset(to_buf, 0, outbytes + 1); ret = iconv(cd, (char**)&from_buf, &inbytes, &pointer, &outbytes); iconv_close(cd); if (ret != -1) { /* conversion succeeded */ octstr_delete(string, 0, octstr_len(string)); octstr_append_data(string, to_buf, pointer - to_buf); if (ret) debug("charset", 0, "charset_convert did %d non-reversible conversions", ret); ret = 0; } else { error(0,"Failed to convert string from <%s> to <%s>, errno was <%d>", charset_from, charset_to, errno); } if (errno == EILSEQ) { debug("charset_convert", 0, "Found an invalid multibyte sequence at position <%d>", from_buf - octstr_get_cstr(string)); } gw_free(to_buf); return ret; #endif /* no convertion done due to not having iconv */ return -1; }

Here is the call graph for this function:

int charset_from_utf8 (  Octstr *  utf8, Octstr **  to, Octstr *  charset_to )

Definition at line 559 of file charset.c. References handler, octstr_create_from_data, octstr_get_cstr, and octstr_len(). Referenced by octstr_recode(). { int ret; xmlCharEncodingHandlerPtr handler = NULL; xmlBufferPtr frombuffer = NULL; xmlBufferPtr tobuffer = NULL; handler = xmlFindCharEncodingHandler(octstr_get_cstr(charset_to)); if (handler == NULL) return -2; /* Build the libxml buffers for the transcoding. */ tobuffer = xmlBufferCreate(); frombuffer = xmlBufferCreate(); xmlBufferAdd(frombuffer, (unsigned char*)octstr_get_cstr(utf8), octstr_len(utf8)); ret = xmlCharEncOutFunc(handler, tobuffer, frombuffer); if (ret < -2) /* Libxml seems to be here a little uncertain what would be the * return code -3, so let's make it -1. Ugly thing, indeed. --tuo */ ret = -1; *to = octstr_create_from_data((char*)tobuffer->content, tobuffer->use); /* Memory cleanup. */ xmlBufferFree(tobuffer); xmlBufferFree(frombuffer); return ret; }

Here is the call graph for this function:

void charset_gsm_to_latin1 (  Octstr *  ostr  )

Definition at line 395 of file charset.c. References gsm_esctolatin1, gsm_to_latin1, octstr_delete(), octstr_get_char(), octstr_len(), and octstr_set_char(). Referenced by main(). { long pos, len; len = octstr_len(ostr); for (pos = 0; pos < len; pos++) { int c, new, i; c = octstr_get_char(ostr, pos); if (c == 27 && pos + 1 < len) { /* GSM escape code. Delete it, then process the next * character specially. */ octstr_delete(ostr, pos, 1); len--; c = octstr_get_char(ostr, pos); for (i = 0; gsm_esctolatin1[i].gsmesc >= 0; i++) { if (gsm_esctolatin1[i].gsmesc == c) break; } if (gsm_esctolatin1[i].gsmesc == c) new = gsm_esctolatin1[i].latin1; else if (c < 128) new = gsm_to_latin1[c]; else continue; } else if (c < 128) { new = gsm_to_latin1[c]; } else { continue; } if (new != c) octstr_set_char(ostr, pos, new); } }

Here is the call graph for this function:

void charset_gsm_to_nrc_iso_21_german (  Octstr *  ostr  )

Definition at line 461 of file charset.c. References octstr_get_char(), octstr_len(), and octstr_set_char(). Referenced by msg_to_emimsg(). { long pos, len; int c, new; len = octstr_len(ostr); for (pos = 0; pos < len; pos++) { c = octstr_get_char(ostr, pos); switch (c) { /* GSM value; NRC value */ case 0x5b: new = 0x5b; break; /* Ä */ case 0x5c: new = 0x5c; break; /* Ö */ case 0x5e: new = 0x5d; break; /* Ü */ case 0x7b: new = 0x7b; break; /* ä */ case 0x7c: new = 0x7c; break; /* ö */ case 0x7e: new = 0x7d; break; /* ü */ case 0x1e: new = 0x7e; break; /* ß */ case 0x5f: new = 0x5e; break; /* § */ default: new = c; } if (new != c) octstr_set_char(ostr, pos, new); } }

Here is the call graph for this function:

void charset_gsm_to_utf8 (  Octstr *  ostr  )

Convert octet string in GSM format to UTF-8. Every GSM character can be represented with unicode, hence nothing will be lost. Escaped charaters will be translated into appropriate UTF-8 character. Definition at line 221 of file charset.c. References gsm_esctouni, gsm_to_unicode, octstr_append(), octstr_append_char(), octstr_create, octstr_destroy(), octstr_get_char(), octstr_len(), octstr_truncate(), and warning(). Referenced by at2_decode7bituncompressed(), cimd2_accept_message(), convert_addr_from_pdu(), data_sm_to_msg(), extract_msgdata_part_by_coding(), handle_operation(), main(), oisd_accept_message(), and pdu_to_msg(). { long pos, len; Octstr *newostr; if (ostr == NULL) return; newostr = octstr_create(""); len = octstr_len(ostr); for (pos = 0; pos < len; pos++) { int c, i; c = octstr_get_char(ostr, pos); if (c > 127) { warning(0, "Could not convert GSM (0x%02x) to Unicode.", c); continue; } if(c == 27 && pos + 1 < len) { c = octstr_get_char(ostr, ++pos); for (i = 0; gsm_esctouni[i].gsmesc >= 0; i++) { if (gsm_esctouni[i].gsmesc == c) break; } if (gsm_esctouni[i].gsmesc == c) { /* found a value for escaped char */ c = gsm_esctouni[i].unichar; } else { /* nothing found, look esc in our table */ c = gsm_to_unicode[27]; pos--; } } else if (c < 128) { c = gsm_to_unicode[c]; } /* unicode to utf-8 */ if(c < 128) { /* 0-127 are ASCII chars that need no conversion */ octstr_append_char(newostr, c); } else { /* test if it can be converterd into a two byte char */ if(c < 0x0800) { octstr_append_char(newostr, ((c >> 6) | 0xC0) & 0xFF); /* add 110xxxxx */ octstr_append_char(newostr, (c & 0x3F) | 0x80); /* add 10xxxxxx */ } else { /* else we encode with 3 bytes. This only happens in case of euro symbol */ octstr_append_char(newostr, ((c >> 12) | 0xE0) & 0xFF); /* add 1110xxxx */ octstr_append_char(newostr, (((c >> 6) & 0x3F) | 0x80) & 0xFF); /* add 10xxxxxx */ octstr_append_char(newostr, ((c & 0x3F) | 0x80) & 0xFF); /* add 10xxxxxx */ } /* There are no 4 bytes encoded characters in GSM charset */ } } octstr_truncate(ostr, 0); octstr_append(ostr, newostr); octstr_destroy(newostr); }

Here is the call graph for this function:

int charset_gsm_truncate (  Octstr *  gsm, long  max )

Definition at line 513 of file charset.c. References gsm, octstr_get_char(), octstr_len(), and octstr_truncate(). Referenced by extract_msgdata_part_by_coding(), msg_to_emimsg(), and packet_encode_message(). { if (octstr_len(gsm) > max) { /* If the last GSM character was an escaped character, * then chop off the escape as well as the character. */ if (octstr_get_char(gsm, max - 1) == 27) octstr_truncate(gsm, max - 1); else octstr_truncate(gsm, max); return 1; } return 0; }

Here is the call graph for this function:

void charset_init (  void   )

Definition at line 201 of file charset.c. References chars_aliases, and alias_t::real. Referenced by gwlib_init(). { int i; for (i = 0; chars_aliases[i].real != NULL; i++) { xmlAddEncodingAlias(chars_aliases[i].real,chars_aliases[i].alias); /*debug("encoding",0,"Add encoding for %s",chars_aliases[i].alias);*/ } }

void charset_latin1_to_gsm (  Octstr *  ostr  )

Definition at line 431 of file charset.c. References gw_assert, octstr_get_char(), octstr_insert_data(), octstr_len(), and octstr_set_char(). { long pos, len; int c, new; unsigned char esc = 27; len = octstr_len(ostr); for (pos = 0; pos < len; pos++) { c = octstr_get_char(ostr, pos); gw_assert(c >= 0); gw_assert(c <= 256); new = latin1_to_gsm[c]; if (new < 0) { /* Escaped GSM code */ octstr_insert_data(ostr, pos, (char*) &esc, 1); pos++; len++; new = -new; } if (new != c) octstr_set_char(ostr, pos, new); } }

Here is the call graph for this function:

void charset_nrc_iso_21_german_to_gsm (  Octstr *  ostr  )

Definition at line 487 of file charset.c. References octstr_get_char(), octstr_len(), and octstr_set_char(). Referenced by handle_operation(). { long pos, len; int c, new; len = octstr_len(ostr); for (pos = 0; pos < len; pos++) { c = octstr_get_char(ostr, pos); switch (c) { /* NRC value; GSM value */ case 0x5b: new = 0x5b; break; /* Ä */ case 0x5c: new = 0x5c; break; /* Ö */ case 0x5d: new = 0x5e; break; /* Ü */ case 0x7b: new = 0x7b; break; /* ä */ case 0x7c: new = 0x7c; break; /* ö */ case 0x7d: new = 0x7e; break; /* ü */ case 0x7e: new = 0x1e; break; /* ß */ case 0x5e: new = 0x5f; break; /* § */ default: new = c; } if (new != c) octstr_set_char(ostr, pos, new); } }

Here is the call graph for this function:

void charset_shutdown (  void   )

Definition at line 211 of file charset.c. Referenced by gwlib_shutdown(). { xmlCleanupEncodingAliases(); }

int charset_to_utf8 (  Octstr *  from, Octstr **  to, Octstr *  charset_from )

Definition at line 527 of file charset.c. References handler, octstr_compare(), octstr_create_from_data, octstr_duplicate, octstr_get_cstr, octstr_imm(), and octstr_len(). Referenced by octstr_recode(), and set_charset(). { int ret; xmlCharEncodingHandlerPtr handler = NULL; xmlBufferPtr frombuffer = NULL; xmlBufferPtr tobuffer = NULL; if (octstr_compare(charset_from, octstr_imm("UTF-8")) == 0) { *to = octstr_duplicate(from); return 0; } handler = xmlFindCharEncodingHandler(octstr_get_cstr(charset_from)); if (handler == NULL) return -2; /* Build the libxml buffers for the transcoding. */ tobuffer = xmlBufferCreate(); frombuffer = xmlBufferCreate(); xmlBufferAdd(frombuffer, (unsigned char*)octstr_get_cstr(from), octstr_len(from)); ret = xmlCharEncInFunc(handler, tobuffer, frombuffer); *to = octstr_create_from_data((char*)tobuffer->content, tobuffer->use); /* Memory cleanup. */ xmlBufferFree(tobuffer); xmlBufferFree(frombuffer); return ret; }

Here is the call graph for this function:

void charset_utf8_to_gsm (  Octstr *  ostr  )

Convert octet string in UTF-8 format to GSM 03.38. Because not all UTF-8 charater can be converted to GSM 03.38 non convertable character replaces with NRP character (see define above). Special characters will be formed into escape sequences. Incomplete UTF-8 characters at the end of the string will be skipped. Definition at line 289 of file charset.c. References octstr_append(), octstr_append_char(), octstr_create, octstr_destroy(), octstr_get_char(), octstr_len(), octstr_truncate(), and warning(). Referenced by at2_pdu_encode(), extract_msgdata_part_by_coding(), main(), msg_to_emimsg(), msg_to_pdu(), ois_append_sm_text(), packet_encode_message(), and sms_msgdata_len(). { long pos, len; int val1, val2; Octstr *newostr; if (ostr == NULL) return; newostr = octstr_create(""); len = octstr_len(ostr); for (pos = 0; pos < len; pos++) { val1 = octstr_get_char(ostr, pos); /* check range */ if (val1 < 0 || val1 > 255) { warning(0, "Char (0x%02x) in UTF-8 string not in the range (0, 255). Skipped.", val1); continue; } /* Convert UTF-8 to unicode code */ /* test if two byte utf8 char */ if ((val1 & 0xE0) == 0xC0) { /* test if incomplete utf char */ if(pos + 1 < len) { val2 = octstr_get_char(ostr, ++pos); val1 = (((val1 & ~0xC0) << 6) | (val2 & 0x3F)); } else { /* incomplete, ignore it */ warning(0, "Incomplete UTF-8 char discovered, skipped. 1"); pos += 1; continue; } } else if ((val1 & 0xF0) == 0xE0) { /* test for three byte utf8 char */ if(pos + 2 < len) { val2 = octstr_get_char(ostr, ++pos); val1 = (((val1 & ~0xE0) << 6) | (val2 & 0x3F)); val2 = octstr_get_char(ostr, ++pos); val1 = (val1 << 6) | (val2 & 0x3F); } else { /* incomplete, ignore it */ warning(0, "Incomplete UTF-8 char discovered, skipped. 2"); pos += 2; continue; } } /* test Latin code page 1 char */ if(val1 <= 255) { val1 = latin1_to_gsm[val1]; /* needs to be escaped ? */ if(val1 < 0) { octstr_append_char(newostr, 27); val1 *= -1; } } else { /* Its not a Latin1 char, test for allowed GSM chars */ switch(val1) { case 0x394: val1 = 0x10; /* GREEK CAPITAL LETTER DELTA */ break; case 0x3A6: val1 = 0x12; /* GREEK CAPITAL LETTER PHI */ break; case 0x393: val1 = 0x13; /* GREEK CAPITAL LETTER GAMMA */ break; case 0x39B: val1 = 0x14; /* GREEK CAPITAL LETTER LAMBDA */ break; case 0x3A9: val1 = 0x15; /* GREEK CAPITAL LETTER OMEGA */ break; case 0x3A0: val1 = 0x16; /* GREEK CAPITAL LETTER PI */ break; case 0x3A8: val1 = 0x17; /* GREEK CAPITAL LETTER PSI */ break; case 0x3A3: val1 = 0x18; /* GREEK CAPITAL LETTER SIGMA */ break; case 0x398: val1 = 0x19; /* GREEK CAPITAL LETTER THETA */ break; case 0x39E: val1 = 0x1A; /* GREEK CAPITAL LETTER XI */ break; case 0x20AC: val1 = 'e'; /* EURO SIGN */ octstr_append_char(newostr, 27); break; default: val1 = NRP; /* character cannot be represented in GSM 03.38 */ } } octstr_append_char(newostr, val1); } octstr_truncate(ostr, 0); octstr_append(ostr, newostr); octstr_destroy(newostr); }

Here is the call graph for this function:

---

Variable Documentation

alias_t chars_aliases[]

Initial value: { { "CP1250", "WIN-1250" }, { "CP1250", "WINDOWS-1250" }, { "CP1251", "WIN-1251" }, { "CP1251", "WINDOWS-1251" }, { "CP1252", "WIN-1252" }, { "CP1252", "WINDOWS-1252" }, { "CP1253", "WIN-1253" }, { "CP1253", "WINDOWS-1253" }, { "CP1254", "WIN-1254" }, { "CP1254", "WINDOWS-1254" }, { "CP1257", "WIN-1257" }, { "CP1257", "WINDOWS-1257" }, { NULL } } Definition at line 185 of file charset.c. Referenced by charset_init().

const { ... } gsm_esctolatin1[] [static]

Referenced by charset_gsm_to_latin1().

const { ... } gsm_esctouni[] [static]

Struct maps escaped GSM chars to unicode codeposition. Referenced by charset_gsm_to_utf8().

const unsigned char gsm_to_latin1[128] [static]

Initial value: { '@', 0xa3, '$', 0xa5, 0xe8, 0xe9, 0xf9, 0xec, 0xf2, 0xc7, 10, 0xd8, 0xf8, 13, 0xc5, 0xe5, '?', '_', '?', '?', '?', '?', '?', '?', '?', '?', '?', ' ', 0xc6, 0xe6, 0xdf, 0xc9, ' ', '!', '"', '#', 0xa4, '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', 0xa1, 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 0xc4, 0xd6, 0xd1, 0xdc, 0xa7, 0xbf, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 0xe4, 0xf6, 0xf1, 0xfc, 0xe0 } Definition at line 128 of file charset.c. Referenced by charset_gsm_to_latin1().

const int gsm_to_unicode[128] [static]

Initial value: { '@', 0xA3, '$', 0xA5, 0xE8, 0xE9, 0xF9, 0xEC, 0xF2, 0xC7, 10, 0xd8, 0xF8, 13, 0xC5, 0xE5, 0x394, '_', 0x3A6, 0x393, 0x39B, 0x3A9, 0x3A0, 0x3A8, 0x3A3, 0x398, 0x39E, NRP, 0xC6, 0xE6, 0xDF, 0xC9, ' ', '!', '"', '#', 0xA4, '%', '&', '\'', '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', 0xA1, 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', 0xC4, 0xD6, 0xD1, 0xDC, 0xA7, 0xBF, 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', 0xE4, 0xF6, 0xF1, 0xFC, 0xE0 } Map GSM default alphabet characters to unicode codeposition. The escape character, at position 27, is mapped to a NRP, though normally the function that indexes into this table will treat it specially. Definition at line 154 of file charset.c. Referenced by charset_gsm_to_utf8().

int gsmesc

Definition at line 105 of file charset.c.

int latin1

Definition at line 85 of file charset.c. Referenced by convert_html_entity(), and ws_bc_encode().

int unichar

Definition at line 106 of file charset.c.

See file LICENSE for details about the license agreement for using, modifying, copying or deriving work from this software.