News reading now outputs utf-8

lib/charset.c

@@ -40,6 +40,7 @@
 struct _charalias charalias[] = {
     {(char *)"ASCII",           (char *)"US-ASCII"},
     {(char *)"VT100",           (char *)"US-ASCII"},
+    {(char *)"LATIN",		(char *)"LATIN-1"},
     {(char *)"AMIGA",           (char *)"CP437"},
     {(char *)"IBMPC",           (char *)"CP437"},
     {(char *)"PC-8",            (char *)"CP437"},
@@ -77,7 +78,7 @@ struct _charmap charmap[] = {
     {FTNC_CP895,  (char *)"CP895 2",  (char *)"iso-8859-2", (char *)"CP895",    (char *)"ISO-8859-2", (char *)"cs_CZ",       (char *)"IBM codepage 895 (Czech, Kamenicky)"},
     {FTNC_LATIN_5,(char *)"LATIN-5 2",(char *)"iso-8859-5", (char *)"LATIN5",   (char *)"ISO-8859-5", (char *)"turks",       (char *)"ISO 8859-5 (Turkish)"},
     {FTNC_CP866,  (char *)"CP866 2",  (char *)"iso-8859-5", (char *)"CP866",    (char *)"ISO-8859-5", (char *)"ru_RU",       (char *)"IBM codepage 866 (Russian)"},
-    {FTNC_LATIN_9,(char *)"LATIN-9 2",(char *)"iso-8859-15",(char *)"LATIN9",   (char *)"ISO-8859-15",(char *)"en_US",       (char *)"ISO 8859-1 (Western European EURO)"},
+    {FTNC_LATIN_9,(char *)"LATIN-9 2",(char *)"iso-8859-15",(char *)"LATIN-9",  (char *)"ISO-8859-15",(char *)"en_US",       (char *)"ISO 8859-1 (Western European EURO)"},
     {FTNC_KOI8_R, (char *)"KOI8-R 2", (char *)"koi8-r",     (char *)"KOI8-R",   (char *)"KOI8-R",     (char *)"ru_RUi.koi8r",(char *)"Unix codepage KOI8-R (Russian)"},
     {FTNC_CP936,  (char *)"CP936 2",  (char *)"hz-gb-2312", (char *)"GB2312",   (char *)"GB2312",     (char *)"zh_CN.gbk",   (char *)"IBM codepage 936 (Chinese, GBK)"},
     {FTNC_ERROR,  NULL,               NULL,                 NULL,               NULL,                 NULL,                  (char *)"ERROR"}
@@ -138,15 +139,14 @@ int find_ftn_charset(char *ftnkludge)
 	    break;
     }
 
-    if (charalias[i].alias == NULL) {
+    if (charalias[i].alias != NULL) {
-	Syslog('n', "no alias found");
-    } else {
 	Syslog('n', "found alias %s", charalias[i].ftnkludge);
 	snprintf(ftn, 80, "%s", charalias[i].ftnkludge);
     }
 
     /*
-     * Now search real entry
+     * Now search real entry. Throw away the charset level number,
+     * we don't care about that useless byte.
      */
     for (i = 0; charmap[i].ftnkludge; i++) {
 	snprintf(cmp, 80, "%s", charmap[i].ftnkludge);
@@ -165,10 +165,10 @@ int find_ftn_charset(char *ftnkludge)
 
     if (charmap[i].ftnkludge == NULL) {
 	WriteError("find_ftn_charset(%s) not found", ftnkludge);
-	return -1;
+	return FTNC_ERROR;
     }
 
-    Syslog('n', "get_rfc_charset(%s) result %d", ftnkludge, i);
+    Syslog('n', "find_ftn_charset(%s) result %d", ftnkludge, i);
     return i;
 }
 
@@ -242,6 +242,9 @@ char *getftnchrs(int val)
 	}
     }
 
+    /*
+     * Not found, return a default.
+     */
     return (char *)"LATIN-1 2";
 }
 
@@ -259,6 +262,9 @@ char *getrfcchrs(int val)
 	}
     }
 
+    /*
+     * Not found, return a default
+     */
     return (char *)"iso-8859-1";
 }
 

mbnntp/commands.c

@@ -64,34 +64,31 @@ void send_xlat(char *);
 void send_xlat(char *inp)
 {
     char    	temp[1024];
-    int     i;
+    size_t	rc, inSize, outSize;
-#ifndef	USE_EXPERIMENT
-    char    *xl;
-#endif
 
+    if (use_iconv) {
+
+	inSize = strlen(inp);
+	outSize = sizeof(temp);
+	char* in = inp;
+	char* out = temp;
 	memset(&temp, 0, sizeof(temp));
 
-    for (i = 0; i < strlen(inp); i++) {
+	rc = iconv(iconv_s, &in, &inSize, &out, &outSize);
-#ifndef	USE_EXPERIMENT
+	if (rc == -1) {
-	if (inp[i] & 0x80) {
+	    WriteError("$iconv(%s)", printable(inp, 0));
-	    if ((xl = charset_map_c(inp[i], FALSE))) {
+	    strncpy(temp, inp, 1023);
-		while (*xl) {
-		    temp[i] = *xl++;
-		    if (*xl)
-			i++;
-		}
-	    }
-	} else {
-	    temp[i] = inp[i];
-	}
-#else
-	temp[i] = inp[i];
-#endif
 	}
 
+	if (strcmp(inp, temp))
     	    Syslog('n', "i \"%s\"", printable(inp, 0));
     	Syslog('n', "> \"%s\"", printable(temp, 0));
     	PUTSTR(temp);
+    } else {
+	Syslog('n', "> \"%s\"", printable(inp, 0));
+	PUTSTR(inp);
+    }
+
     PUTSTR((char *)"\r\n");
     FLUSHOUT();
     sentbytes += (strlen(temp) + 2);
@@ -183,7 +180,7 @@ void command_abhs(char *buf)
 	return;
     }
 
-    if (Msg_Read(art, 75)) {
+    if (Msg_Read(art, 80)) {
 
 	if (strcasecmp(cmd, "ARTICLE") == 0)
 	    send_nntp("220 %ld %s Article retrieved - Head and body follow", art, make_msgid(Msg.Msgid));
@@ -208,9 +205,9 @@ void command_abhs(char *buf)
 
 	if (charset == NULL) {
 	    if (msgs.Charset != FTNC_NONE) {
-		charset = xstrcpy(getrfcchrs(msgs.Charset));
+		charset = xstrcpy(getftnchrs(msgs.Charset));
 	    } else if (usercharset != FTNC_NONE) {
-		charset = xstrcpy(getrfcchrs(msgs.Charset));
+		charset = xstrcpy(getftnchrs(msgs.Charset));
 	    } else {
 		charset = xstrcpy((char *)"CP437");
 	    }
@@ -220,17 +217,18 @@ void command_abhs(char *buf)
 	charindex = find_ftn_charset(charset);
 
 	if (charindex != FTNC_ERROR) {
-	    Syslog('n', "setup iconv for %s to %s", charmap[charindex].ic_ftn, charmap[charindex].ic_rfc);
+	    Syslog('n', "iconv_open(UTF-8, %s)", charmap[charindex].ic_ftn);
-	    iconv_s = iconv_open(charmap[charindex].ic_rfc, charmap[charindex].ic_ftn);
+//	    Syslog('n', "iconv_open(%s, %s)", charmap[charindex].ic_rfc, charmap[charindex].ic_ftn);
+//	    Syslog('n', "setlocale() result %s", MBSE_SS(setlocale(LC_CTYPE, charmap[charindex].lang)));
+	    iconv_s = iconv_open("UTF-8", charmap[charindex].ic_ftn);
+//	    iconv_s = iconv_open(charmap[charindex].ic_rfc, charmap[charindex].ic_ftn);
 	    if (iconv_s != (iconv_t)-1) {
 		use_iconv = TRUE;
+	    } else {
+		WriteError("$iconv_open(UTF-8, %s)", charmap[charindex].ic_ftn);
 	    }
 	}
 
-
-//	We don't do translation to the users charset, the news reader must do that.
-//	charset_set_in_out(getrfcchrs(msgs.Charset),getrfcchrs(usercharset));
-
 	if ((strcasecmp(cmd, "ARTICLE") == 0) || (strcasecmp(cmd, "HEAD") == 0)) {
 
 	    send_nntp("Path: MBNNTP!not-for-mail");
@@ -252,12 +250,13 @@ void command_abhs(char *buf)
 	     *           4. Default us-ascii.
 	     */
 	    send_nntp("MIME-Version: 1.0");
-	    if (charindex != FTNC_ERROR) {
+//	    if (charindex != FTNC_ERROR) {
-		send_nntp("Content-Type: text/plain; charset=%s", getrfcchrs(charindex));
+//		send_nntp("Content-Type: text/plain; charset=%s", getrfcchrs(charindex));
-	    } else {
+//	    } else {
-		send_nntp("Content-Type: text/plain; charset=us-ascii; format=fixed");
+//		send_nntp("Content-Type: text/plain; charset=us-ascii; format=fixed");
-	    }
+//	    }
-	    send_nntp("Content-Transfer-Encoding: 8bit");
+//	    send_nntp("Content-Transfer-Encoding: 8bit");
+	    send_nntp("Content-Type: text/plain; charset=utf-8");
 
 	    send_nntp("X-FTN-From: %s <%s>", Msg.From, Msg.FromAddress);
 	    if (strlen(Msg.To))

mbsetup/mbsetup.c

@@ -384,7 +384,6 @@ void initdatabases(void)
     InitRoute();
     InitFDB();
     InitIBC();
-
     if (!init) {
 	clr_index();
     }