wp
/
xed
1
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

Merge smart converter in the document output stream 

Based on 85279adad6 (diff-c243a4d5a7789c35057282ee780df3d5)
This commit is contained in:
JosephMcc 2017-01-19 03:29:38 -08:00
parent 2fb3813793
commit e3e9566570
9 changed files with 635 additions and 874 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
tests/Makefile.am
View File

@ -3,11 +3,7 @@ AM_CPPFLAGS = -g -I$(top_srcdir) -I$(top_srcdir)/xed $(XED_DEBUG_FLAGS) $(XED_CF
noinst_PROGRAMS = $(TEST_PROGS)
progs_ldadd = $(top_builddir)/xed/libxed.la
TEST_PROGS = smart-converter
smart_converter_SOURCES = smart-converter.c
smart_converter_LDADD = $(progs_ldadd)
TEST_PROGS += document-input-stream
TEST_PROGS = document-input-stream
document_input_stream_SOURCES = document-input-stream.c
document_input_stream_LDADD = $(progs_ldadd)

239
tests/document-output-stream.c
View File

@ -40,9 +40,11 @@ test_consecutive_write (const gchar *inbuf,
GError *err = NULL;
gchar *b;
XedDocumentNewlineType type;
GSList *encodings = NULL;
doc = xed_document_new ();
out = xed_document_output_stream_new (doc);
encodings = g_slist_prepend (encodings, (gpointer)xed_encoding_get_utf8 ());
out = xed_document_output_stream_new (doc, encodings);
n = 0;
@ -119,6 +121,237 @@ test_big_char ()
XED_DOCUMENT_NEWLINE_TYPE_LF);
}
/* SMART CONVERSION */
#define TEXT_TO_CONVERT "this is some text to make the tests"
#define TEXT_TO_GUESS "hello \xe6\x96\x87 world"
static void
print_hex (gchar *ptr, gint len)
{
gint i;
for (i = 0; i < len; ++i)
{
g_printf ("\\x%02x", (unsigned char)ptr[i]);
}
g_printf ("\n");
}
static gchar *
get_encoded_text (const gchar *text,
gsize nread,
const XedEncoding *to,
const XedEncoding *from,
gsize *bytes_written_aux,
gboolean care_about_error)
{
GCharsetConverter *converter;
gchar *out, *out_aux;
gsize bytes_read, bytes_read_aux;
gsize bytes_written;
GConverterResult res;
GError *err;
converter = g_charset_converter_new (xed_encoding_get_charset (to),
xed_encoding_get_charset (from),
NULL);
out = g_malloc (200);
out_aux = g_malloc (200);
err = NULL;
bytes_read_aux = 0;
*bytes_written_aux = 0;
if (nread == -1)
{
nread = strlen (text);
}
do
{
res = g_converter_convert (G_CONVERTER (converter),
text + bytes_read_aux,
nread,
out_aux,
200,
G_CONVERTER_INPUT_AT_END,
&bytes_read,
&bytes_written,
&err);
memcpy (out + *bytes_written_aux, out_aux, bytes_written);
bytes_read_aux += bytes_read;
*bytes_written_aux += bytes_written;
nread -= bytes_read;
} while (res != G_CONVERTER_FINISHED && res != G_CONVERTER_ERROR);
if (care_about_error)
{
g_assert_no_error (err);
}
else if (err)
{
g_printf ("** You don't care, but there was an error: %s", err->message);
return NULL;
}
out[*bytes_written_aux] = '\0';
if (!g_utf8_validate (out, *bytes_written_aux, NULL) && !care_about_error)
{
if (!care_about_error)
{
return NULL;
}
else
{
g_assert_not_reached ();
}
}
return out;
}
static GSList *
get_all_encodings ()
{
GSList *encs = NULL;
gint i = 0;
while (TRUE)
{
const XedEncoding *enc;
enc = xed_encoding_get_from_index (i);
if (enc == NULL)
break;
encs = g_slist_prepend (encs, (gpointer)enc);
i++;
}
return encs;
}
static gchar *
do_test (const gchar *test_in,
const gchar *enc,
GSList *encodings,
gsize nread,
const XedEncoding **guessed)
{
XedDocument *doc;
GOutputStream *out;
GError *err = NULL;
GtkTextIter start, end;
gchar *text;
if (enc != NULL)
{
encodings = NULL;
encodings = g_slist_prepend (encodings, (gpointer)xed_encoding_get_from_charset (enc));
}
doc = xed_document_new ();
encodings = g_slist_prepend (encodings, (gpointer)xed_encoding_get_utf8 ());
out = xed_document_output_stream_new (doc, encodings);
g_output_stream_write (out, test_in, nread, NULL, &err);
g_assert_no_error (err);
g_output_stream_flush (out, NULL, &err);
g_assert_no_error (err);
g_output_stream_close (out, NULL, &err);
g_assert_no_error (err);
if (guessed != NULL)
*guessed = xed_document_output_stream_get_guessed (XED_DOCUMENT_OUTPUT_STREAM (out));
gtk_text_buffer_get_bounds (GTK_TEXT_BUFFER (doc), &start, &end);
text = gtk_text_buffer_get_text (GTK_TEXT_BUFFER (doc),
&start,
&end,
FALSE);
g_object_unref (doc);
g_object_unref (out);
return text;
}
static void
test_utf8_utf8 ()
{
gchar *aux;
aux = do_test (TEXT_TO_CONVERT, "UTF-8", NULL, strlen (TEXT_TO_CONVERT), NULL);
g_assert_cmpstr (aux, ==, TEXT_TO_CONVERT);
aux = do_test ("foobar\xc3\xa8\xc3\xa8\xc3\xa8zzzzzz", "UTF-8", NULL, 18, NULL);
g_assert_cmpstr (aux, ==, "foobar\xc3\xa8\xc3\xa8\xc3\xa8zzzzzz");
aux = do_test ("foobar\xc3\xa8\xc3\xa8\xc3\xa8zzzzzz", "UTF-8", NULL, 12, NULL);
g_assert_cmpstr (aux, ==, "foobar\xc3\xa8\xc3\xa8\xc3\xa8");
/* FIXME: Use the utf8 stream for a fallback? */
//do_test_with_error ("\xef\xbf\xbezzzzzz", encs, G_IO_ERROR_FAILED);
}
static void
test_empty_conversion ()
{
const XedEncoding *guessed;
gchar *out;
GSList *encodings = NULL;
/* testing the case of an empty file and list of encodings with no
utf-8. In this case, the smart converter cannot determine the right
encoding (because there is no input), but should still default to
utf-8 for the detection */
encodings = g_slist_prepend (encodings, (gpointer)xed_encoding_get_from_charset ("UTF-16"));
encodings = g_slist_prepend (encodings, (gpointer)xed_encoding_get_from_charset ("ISO-8859-15"));
out = do_test ("", NULL, encodings, 0, &guessed);
g_assert_cmpstr (out, ==, "");
g_assert (guessed == xed_encoding_get_utf8 ());
}
static void
test_guessed ()
{
GSList *encs = NULL;
gchar *aux, *aux2, *fail;
gsize aux_len, fail_len;
const XedEncoding *guessed;
aux = get_encoded_text (TEXT_TO_GUESS, -1,
xed_encoding_get_from_charset ("UTF-16"),
xed_encoding_get_from_charset ("UTF-8"),
&aux_len,
TRUE);
fail = get_encoded_text (aux, aux_len,
xed_encoding_get_from_charset ("UTF-8"),
xed_encoding_get_from_charset ("ISO-8859-15"),
&fail_len,
FALSE);
g_assert (fail == NULL);
/* ISO-8859-15 should fail */
encs = g_slist_append (encs, (gpointer)xed_encoding_get_from_charset ("ISO-8859-15"));
encs = g_slist_append (encs, (gpointer)xed_encoding_get_from_charset ("UTF-16"));
aux2 = do_test (aux, NULL, encs, aux_len, &guessed);
g_assert (guessed == xed_encoding_get_from_charset ("UTF-16"));
}
int main (int argc,
char *argv[])
{
@ -130,5 +363,9 @@ int main (int argc,
g_test_add_func ("/document-output-stream/consecutive_tnewline", test_consecutive_tnewline);
g_test_add_func ("/document-output-stream/big-char", test_big_char);
g_test_add_func ("/document-output-stream/smart conversion: utf8-utf8", test_utf8_utf8);
g_test_add_func ("/document-output-stream/smart conversion: guessed", test_guessed);
g_test_add_func ("/document-output-stream/smart conversion: empty", test_empty_conversion);
return g_test_run ();
}

353
tests/smart-converter.c
View File

@ -1,353 +0,0 @@
/*
* smart-converter.c
* This file is part of xed
*
* Copyright (C) 2009 - Ignacio Casal Quinteiro
*
* xed is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* xed is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with xed; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301 USA
*/
#include "xed-smart-charset-converter.h"
#include "xed-encodings.h"
#include <gio/gio.h>
#include <glib.h>
#include <glib/gprintf.h>
#include <string.h>
#define TEXT_TO_CONVERT "this is some text to make the tests"
#define TEXT_TO_GUESS "hello \xe6\x96\x87 world"
static void
print_hex (gchar *ptr, gint len)
{
gint i;
for (i = 0; i < len; ++i)
{
g_printf ("\\x%02x", (unsigned char)ptr[i]);
}
g_printf ("\n");
}
static gchar *
get_encoded_text (const gchar *text,
gsize nread,
const XedEncoding *to,
const XedEncoding *from,
gsize *bytes_written_aux,
gboolean care_about_error)
{
GCharsetConverter *converter;
gchar *out, *out_aux;
gsize bytes_read, bytes_read_aux;
gsize bytes_written;
GConverterResult res;
GError *err;
converter = g_charset_converter_new (xed_encoding_get_charset (to),
xed_encoding_get_charset (from),
NULL);
out = g_malloc (200);
out_aux = g_malloc (200);
err = NULL;
bytes_read_aux = 0;
*bytes_written_aux = 0;
if (nread == -1)
{
nread = strlen (text);
}
do
{
res = g_converter_convert (G_CONVERTER (converter),
text + bytes_read_aux,
nread,
out_aux,
200,
G_CONVERTER_INPUT_AT_END,
&bytes_read,
&bytes_written,
&err);
memcpy (out + *bytes_written_aux, out_aux, bytes_written);
bytes_read_aux += bytes_read;
*bytes_written_aux += bytes_written;
nread -= bytes_read;
} while (res != G_CONVERTER_FINISHED && res != G_CONVERTER_ERROR);
if (care_about_error)
{
g_assert_no_error (err);
}
else if (err)
{
g_printf ("** You don't care, but there was an error: %s", err->message);
return NULL;
}
out[*bytes_written_aux] = '\0';
if (!g_utf8_validate (out, *bytes_written_aux, NULL) && !care_about_error)
{
if (!care_about_error)
{
return NULL;
}
else
{
g_assert_not_reached ();
}
}
return out;
}
static GSList *
get_all_encodings ()
{
GSList *encs = NULL;
gint i = 0;
while (TRUE)
{
const XedEncoding *enc;
enc = xed_encoding_get_from_index (i);
if (enc == NULL)
break;
encs = g_slist_prepend (encs, (gpointer)enc);
i++;
}
return encs;
}
static gchar *
do_test (const gchar *test_in,
const gchar *enc,
GSList *encodings,
gsize nread,
const XedEncoding **guessed)
{
XedSmartCharsetConverter *converter;
gchar *out, *out_aux;
gsize bytes_read, bytes_read_aux;
gsize bytes_written, bytes_written_aux;
GConverterResult res;
GError *err;
if (enc != NULL)
{
encodings = NULL;
encodings = g_slist_prepend (encodings, (gpointer)xed_encoding_get_from_charset (enc));
}
converter = xed_smart_charset_converter_new (encodings);
out = g_malloc (200);
out_aux = g_malloc (200);
err = NULL;
bytes_read_aux = 0;
bytes_written_aux = 0;
do
{
res = g_converter_convert (G_CONVERTER (converter),
test_in + bytes_read_aux,
nread,
out_aux,
200,
G_CONVERTER_INPUT_AT_END,
&bytes_read,
&bytes_written,
&err);
memcpy (out + bytes_written_aux, out_aux, bytes_written);
bytes_read_aux += bytes_read;
bytes_written_aux += bytes_written;
nread -= bytes_read;
} while (res != G_CONVERTER_FINISHED && res != G_CONVERTER_ERROR);
g_assert_no_error (err);
out[bytes_written_aux] = '\0';
if (guessed != NULL)
*guessed = xed_smart_charset_converter_get_guessed (converter);
return out;
}
static void
do_test_roundtrip (const char *str, const char *charset)
{
gsize len;
gchar *buf, *p;
GInputStream *in, *tmp;
GCharsetConverter *c1;
XedSmartCharsetConverter *c2;
gsize n, tot;
GError *err;
GSList *enc = NULL;
len = strlen(str);
buf = g_new0 (char, len);
in = g_memory_input_stream_new_from_data (str, -1, NULL);
c1 = g_charset_converter_new (charset, "UTF-8", NULL);
tmp = in;
in = g_converter_input_stream_new (in, G_CONVERTER (c1));
g_object_unref (tmp);
g_object_unref (c1);
enc = g_slist_prepend (enc, (gpointer)xed_encoding_get_from_charset (charset));
c2 = xed_smart_charset_converter_new (enc);
g_slist_free (enc);
tmp = in;
in = g_converter_input_stream_new (in, G_CONVERTER (c2));
g_object_unref (tmp);
g_object_unref (c2);
tot = 0;
p = buf;
n = len;
while (TRUE)
{
gssize res;
err = NULL;
res = g_input_stream_read (in, p, n, NULL, &err);
g_assert_no_error (err);
if (res == 0)
break;
p += res;
n -= res;
tot += res;
}
g_assert_cmpint (tot, ==, len);
g_assert_cmpstr (str, ==, buf);
g_free (buf);
g_object_unref (in);
}
static void
test_utf8_utf8 ()
{
gchar *aux;
aux = do_test (TEXT_TO_CONVERT, "UTF-8", NULL, strlen (TEXT_TO_CONVERT), NULL);
g_assert_cmpstr (aux, ==, TEXT_TO_CONVERT);
aux = do_test ("foobar\xc3\xa8\xc3\xa8\xc3\xa8zzzzzz", "UTF-8", NULL, 18, NULL);
g_assert_cmpstr (aux, ==, "foobar\xc3\xa8\xc3\xa8\xc3\xa8zzzzzz");
aux = do_test ("foobar\xc3\xa8\xc3\xa8\xc3\xa8zzzzzz", "UTF-8", NULL, 9, NULL);
g_assert_cmpstr (aux, ==, "foobar\xc3\xa8\xc3");
/* FIXME: Use the utf8 stream for a fallback? */
//do_test_with_error ("\xef\xbf\xbezzzzzz", encs, G_IO_ERROR_FAILED);
}
static void
test_xxx_xxx ()
{
GSList *encs, *l;
encs = get_all_encodings ();
/* Here we just test all encodings it is just to know that the conversions
are done ok */
for (l = encs; l != NULL; l = g_slist_next (l))
{
do_test_roundtrip (TEXT_TO_CONVERT, xed_encoding_get_charset ((const XedEncoding *)l->data));
}
g_slist_free (encs);
}
static void
test_empty ()
{
const XedEncoding *guessed;
gchar *out;
GSList *encodings = NULL;
/* testing the case of an empty file and list of encodings with no
utf-8. In this case, the smart converter cannot determine the right
encoding (because there is no input), but should still default to
utf-8 for the detection */
encodings = g_slist_prepend (encodings, (gpointer)xed_encoding_get_from_charset ("UTF-16"));
encodings = g_slist_prepend (encodings, (gpointer)xed_encoding_get_from_charset ("ISO-8859-15"));
out = do_test ("", NULL, encodings, 0, &guessed);
g_assert_cmpstr (out, ==, "");
g_assert (guessed == xed_encoding_get_utf8 ());
}
static void
test_guessed ()
{
GSList *encs = NULL;
gchar *aux, *aux2, *fail;
gsize aux_len, fail_len;
const XedEncoding *guessed;
aux = get_encoded_text (TEXT_TO_GUESS, -1,
xed_encoding_get_from_charset ("UTF-16"),
xed_encoding_get_from_charset ("UTF-8"),
&aux_len,
TRUE);
fail = get_encoded_text (aux, aux_len,
xed_encoding_get_from_charset ("UTF-8"),
xed_encoding_get_from_charset ("ISO-8859-15"),
&fail_len,
FALSE);
g_assert (fail == NULL);
/* ISO-8859-15 should fail */
encs = g_slist_append (encs, (gpointer)xed_encoding_get_from_charset ("ISO-8859-15"));
encs = g_slist_append (encs, (gpointer)xed_encoding_get_from_charset ("UTF-16"));
aux2 = do_test (aux, NULL, encs, aux_len, &guessed);
g_assert (guessed == xed_encoding_get_from_charset ("UTF-16"));
}
int main (int argc,
char *argv[])
{
g_test_init (&argc, &argv, NULL);
g_test_add_func ("/smart-converter/utf8-utf8", test_utf8_utf8);
//g_test_add_func ("/smart-converter/xxx-xxx", test_xxx_xxx);
g_test_add_func ("/smart-converter/guessed", test_guessed);
g_test_add_func ("/smart-converter/empty", test_empty);
return g_test_run ();
}

2
xed/Makefile.am
View File

@ -59,7 +59,6 @@ NOINST_H_FILES = \
xed-print-preview.h \
xed-session.h \
xed-settings.h \
xed-smart-charset-converter.h \
xed-style-scheme-manager.h \
xed-tab-label.h \
xedtextregion.h \
@ -140,7 +139,6 @@ libxed_c_files = \
xed-progress-message-area.c \
xed-session.c \
xed-settings.c \
xed-smart-charset-converter.c \
xed-searchbar.c \
xed-statusbar.c \
xed-status-combo-box.c \

32
xed/xed-document-loader.c
View File

@ -39,7 +39,6 @@
#include "xed-document-loader.h"
#include "xed-document-output-stream.h"
#include "xed-smart-charset-converter.h"
#include "xed-debug.h"
#include "xed-metadata-manager.h"
#include "xed-utils.h"
@ -112,7 +111,6 @@ struct _XedDocumentLoaderPrivate
GCancellable *cancellable;
GInputStream *stream;
GOutputStream *output;
XedSmartCharsetConverter *converter;
gchar buffer[READ_CHUNK_SIZE];
@ -206,12 +204,6 @@ xed_document_loader_dispose (GObject *object)
priv->output = NULL;
}
if (priv->converter != NULL)
{
g_object_unref (priv->converter);
priv->converter = NULL;
}
if (priv->error != NULL)
{
g_error_free (priv->error);
@ -305,7 +297,6 @@ xed_document_loader_init (XedDocumentLoader *loader)
loader->priv->used = FALSE;
loader->priv->auto_detected_newline_type = XED_DOCUMENT_NEWLINE_TYPE_DEFAULT;
loader->priv->converter = NULL;
loader->priv->error = NULL;
loader->priv->enc_settings = g_settings_new ("org.x.editor.preferences.encodings");
}
@ -544,7 +535,11 @@ async_read_cb (GInputStream *stream,
/* end of the file, we are done! */
if (async->read == 0)
{
loader->priv->auto_detected_encoding = xed_smart_charset_converter_get_guessed (loader->priv->converter);
/* flush the stream to ensure proper line ending detection */
g_output_stream_flush (loader->priv->output, NULL, NULL);
loader->priv->auto_detected_encoding =
xed_document_output_stream_get_guessed (XED_DOCUMENT_OUTPUT_STREAM (loader->priv->output));
loader->priv->auto_detected_newline_type =
xed_document_output_stream_detect_newline_type (XED_DOCUMENT_OUTPUT_STREAM (loader->priv->output));
@ -552,7 +547,7 @@ async_read_cb (GInputStream *stream,
/* Check if we needed some fallback char, if so, check if there was
a previous error and if not set a fallback used error */
/* FIXME Uncomment this when we want to manage conversion fallback */
/*if ((xed_smart_charset_converter_get_num_fallbacks (loader->priv->converter) != 0) &&
/*if ((xed_document_output_stream_get_num_fallbacks (XED_DOCUMENT_OUTPUT_STREAM (loader->priv->ouput)) != 0) &&
loader->priv->error == NULL)
{
g_set_error_literal (&loader->priv->error,
@ -631,6 +626,10 @@ finish_query_info (AsyncData *async)
return;
}
conv_stream = g_object_ref (loader->priv->stream);
g_object_unref (loader->priv->stream);
loader->priv->stream = conv_stream;
/* Get the candidate encodings */
if (loader->priv->encoding == NULL)
{
@ -641,16 +640,9 @@ finish_query_info (AsyncData *async)
candidate_encodings = g_slist_prepend (NULL, (gpointer) loader->priv->encoding);
}
loader->priv->converter = xed_smart_charset_converter_new (candidate_encodings);
g_slist_free (candidate_encodings);
conv_stream = g_converter_input_stream_new (loader->priv->stream, G_CONVERTER (loader->priv->converter));
g_object_unref (loader->priv->stream);
loader->priv->stream = conv_stream;
/* Output stream */
loader->priv->output = xed_document_output_stream_new (loader->priv->document);
loader->priv->output = xed_document_output_stream_new (loader->priv->document, candidate_encodings);
g_slist_free (candidate_encodings);
/* start reading */
read_file_chunk (async);

381
xed/xed-document-output-stream.c
View File

@ -26,7 +26,9 @@
#include <glib.h>
#include <glib/gi18n.h>
#include <gio/gio.h>
#include <errno.h>
#include "xed-document-output-stream.h"
#include "xed-debug.h"
/* NOTE: never use async methods on this stream, the stream is just
* a wrapper around GtkTextBuffer api so that we can use GIO Stream
@ -48,6 +50,16 @@ struct _XedDocumentOutputStreamPrivate
gchar *buffer;
gsize buflen;
/* Encoding detection */
GIConv iconv;
GCharsetConverter *charset_conv;
GSList *encodings;
GSList *current_encoding;
guint is_utf8 : 1;
guint use_first : 1;
guint is_initialized : 1;
guint is_closed : 1;
};
@ -114,12 +126,33 @@ xed_document_output_stream_get_property (GObject *object,
}
}
static void
xed_document_output_stream_dispose (GObject *object)
{
XedDocumentOutputStream *stream = XED_DOCUMENT_OUTPUT_STREAM (object);
if (stream->priv->iconv != NULL)
{
g_iconv_close (stream->priv->iconv);
stream->priv->iconv = NULL;
}
if (stream->priv->charset_conv != NULL)
{
g_object_unref (stream->priv->charset_conv);
stream->priv->charset_conv = NULL;
}
G_OBJECT_CLASS (xed_document_output_stream_parent_class)->dispose (object);
}
static void
xed_document_output_stream_finalize (GObject *object)
{
XedDocumentOutputStream *stream = XED_DOCUMENT_OUTPUT_STREAM (object);
g_free (stream->priv->buffer);
g_slist_free (stream->priv->encodings);
G_OBJECT_CLASS (xed_document_output_stream_parent_class)->finalize (object);
}
@ -152,6 +185,7 @@ xed_document_output_stream_class_init (XedDocumentOutputStreamClass *klass)
object_class->get_property = xed_document_output_stream_get_property;
object_class->set_property = xed_document_output_stream_set_property;
object_class->dispose = xed_document_output_stream_dispose;
object_class->finalize = xed_document_output_stream_finalize;
object_class->constructed = xed_document_output_stream_constructed;
@ -179,10 +213,194 @@ xed_document_output_stream_init (XedDocumentOutputStream *stream)
stream->priv->buffer = NULL;
stream->priv->buflen = 0;
stream->priv->charset_conv = NULL;
stream->priv->encodings = NULL;
stream->priv->current_encoding = NULL;
stream->priv->is_initialized = FALSE;
stream->priv->is_closed = FALSE;
stream->priv->is_utf8 = FALSE;
stream->priv->use_first = FALSE;
}
static const XedEncoding *
get_encoding (XedDocumentOutputStream *stream)
{
if (stream->priv->current_encoding == NULL)
{
stream->priv->current_encoding = stream->priv->encodings;
}
else
{
stream->priv->current_encoding = g_slist_next (stream->priv->current_encoding);
}
if (stream->priv->current_encoding != NULL)
{
return (const XedEncoding *)stream->priv->current_encoding->data;
}
return NULL;
}
static gboolean
try_convert (GCharsetConverter *converter,
const void *inbuf,
gsize inbuf_size)
{
GError *err;
gsize bytes_read, nread;
gsize bytes_written, nwritten;
GConverterResult res;
gchar *out;
gboolean ret;
gsize out_size;
if (inbuf == NULL || inbuf_size == 0)
{
return FALSE;
}
err = NULL;
nread = 0;
nwritten = 0;
out_size = inbuf_size * 4;
out = g_malloc (out_size);
do
{
res = g_converter_convert (G_CONVERTER (converter),
(gchar *)inbuf + nread,
inbuf_size - nread,
(gchar *)out + nwritten,
out_size - nwritten,
G_CONVERTER_INPUT_AT_END,
&bytes_read,
&bytes_written,
&err);
nread += bytes_read;
nwritten += bytes_written;
} while (res != G_CONVERTER_FINISHED && res != G_CONVERTER_ERROR && err == NULL);
if (err != NULL)
{
if (err->code == G_CONVERT_ERROR_PARTIAL_INPUT)
{
/* FIXME We can get partial input while guessing the
encoding because we just take some amount of text
to guess from. */
ret = TRUE;
}
else
{
ret = FALSE;
}
g_error_free (err);
}
else
{
ret = TRUE;
}
/* FIXME: Check the remainder? */
if (ret == TRUE && !g_utf8_validate (out, nwritten, NULL))
{
ret = FALSE;
}
g_free (out);
return ret;
}
static GCharsetConverter *
guess_encoding (XedDocumentOutputStream *stream,
const void *inbuf,
gsize inbuf_size)
{
GCharsetConverter *conv = NULL;
if (inbuf == NULL || inbuf_size == 0)
{
stream->priv->is_utf8 = TRUE;
return NULL;
}
if (stream->priv->encodings != NULL && stream->priv->encodings->next == NULL)
{
stream->priv->use_first = TRUE;
}
/* We just check the first block */
while (TRUE)
{
const XedEncoding *enc;
if (conv != NULL)
{
g_object_unref (conv);
conv = NULL;
}
/* We get an encoding from the list */
enc = get_encoding (stream);
/* if it is NULL we didn't guess anything */
if (enc == NULL)
{
break;
}
xed_debug_message (DEBUG_UTILS, "trying charset: %s",
xed_encoding_get_charset (stream->priv->current_encoding->data));
if (enc == xed_encoding_get_utf8 ())
{
gsize remainder;
const gchar *end;
if (g_utf8_validate (inbuf, inbuf_size, &end) || stream->priv->use_first)
{
stream->priv->is_utf8 = TRUE;
break;
}
/* Check if the end is less than one char */
remainder = inbuf_size - (end - (gchar *)inbuf);
if (remainder < 6)
{
stream->priv->is_utf8 = TRUE;
break;
}
continue;
}
conv = g_charset_converter_new ("UTF-8", xed_encoding_get_charset (enc), NULL);
/* If we tried all encodings we use the first one */
if (stream->priv->use_first)
{
break;
}
/* Try to convert */
if (try_convert (conv, inbuf, inbuf_size))
{
break;
}
}
if (conv != NULL)
{
g_converter_reset (G_CONVERTER (conv));
}
return conv;
}
static XedDocumentNewlineType
get_newline_type (GtkTextIter *end)
{
@ -214,9 +432,16 @@ get_newline_type (GtkTextIter *end)
}
GOutputStream *
xed_document_output_stream_new (XedDocument *doc)
xed_document_output_stream_new (XedDocument *doc,
GSList *candidate_encodings)
{
return G_OUTPUT_STREAM (g_object_new (XED_TYPE_DOCUMENT_OUTPUT_STREAM, "document", doc, NULL));
XedDocumentOutputStream *stream;
stream = g_object_new (XED_TYPE_DOCUMENT_OUTPUT_STREAM, "document", doc, NULL);
stream->priv->encodings = g_slist_copy (candidate_encodings);
return G_OUTPUT_STREAM (stream);
}
XedDocumentNewlineType
@ -239,6 +464,38 @@ xed_document_output_stream_detect_newline_type (XedDocumentOutputStream *stream)
return type;
}
const XedEncoding *
xed_document_output_stream_get_guessed (XedDocumentOutputStream *stream)
{
g_return_val_if_fail (XED_IS_DOCUMENT_OUTPUT_STREAM (stream), NULL);
if (stream->priv->current_encoding != NULL)
{
return (const XedEncoding *)stream->priv->current_encoding->data;
}
else if (stream->priv->is_utf8 || !stream->priv->is_initialized)
{
/* If it is not initialized we assume that we are trying to convert
the empty string */
return xed_encoding_get_utf8 ();
}
return NULL;
}
guint
xed_document_output_stream_get_num_fallbacks (XedDocumentOutputStream *stream)
{
g_return_val_if_fail (XED_IS_DOCUMENT_OUTPUT_STREAM (stream), FALSE);
if (stream->priv->charset_conv == NULL)
{
return FALSE;
}
return g_charset_converter_get_num_fallbacks (stream->priv->charset_conv) != 0;
}
/* If the last char is a newline, remove it from the buffer (otherwise
GtkTextView shows it as an empty line). See bug #324942. */
static void
@ -287,6 +544,8 @@ xed_document_output_stream_write (GOutputStream *stream,
gboolean freetext = FALSE;
const gchar *end;
gboolean valid;
gsize nvalid;
gsize remainder;
if (g_cancellable_set_error_if_cancelled (cancellable, error))
{
@ -297,6 +556,52 @@ xed_document_output_stream_write (GOutputStream *stream,
if (!ostream->priv->is_initialized)
{
ostream->priv->charset_conv = guess_encoding (ostream, buffer, count);
/* If we still have the previous case is that we didn't guess
anything */
if (ostream->priv->charset_conv == NULL && !ostream->priv->is_utf8)
{
/* FIXME: Add a different domain when we kill xed_convert */
g_set_error_literal (error, XED_DOCUMENT_ERROR,
XED_DOCUMENT_ERROR_ENCODING_AUTO_DETECTION_FAILED,
_("It is not possible to detect the encoding automatically"));
return -1;
}
/* Do not initialize iconv if we are not going to conver anything */
if (!ostream->priv->is_utf8)
{
gchar *from_charset;
/* Initialize iconv */
g_object_get (G_OBJECT (ostream->priv->charset_conv), "from-charset", &from_charset, NULL);
ostream->priv->iconv = g_iconv_open ("UTF-8", from_charset);
if (ostream->priv->iconv == (GIConv)-1)
{
if (errno == EINVAL)
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_NOT_SUPPORTED,
_("Conversion from character set '%s' to 'UTF-8' is not supported"),
from_charset);
}
else
{
g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
_("Could not open converter from '%s' to 'UTF-8'"),
from_charset);
}
g_free (from_charset);
return -1;
}
g_free (from_charset);
}
/* Init the undoable action */
gtk_source_buffer_begin_not_undoable_action (GTK_SOURCE_BUFFER (ostream->priv->doc));
@ -308,12 +613,17 @@ xed_document_output_stream_write (GOutputStream *stream,
{
len = ostream->priv->buflen + count;
text = g_new (gchar , len + 1);
memcpy (text, ostream->priv->buffer, ostream->priv->buflen);
memcpy (text + ostream->priv->buflen, buffer, count);
text[len] = '\0';
g_free (ostream->priv->buffer);
ostream->priv->buffer = NULL;
ostream->priv->buflen = 0;
freetext = TRUE;
}
else
@ -322,8 +632,71 @@ xed_document_output_stream_write (GOutputStream *stream,
len = count;
}
if (!ostream->priv->is_utf8)
{
gchar *conv_text;
gsize conv_read;
gsize conv_written;
GError *err = NULL;
if (ostream->priv->iconv == NULL)
{
g_set_error_literal (error, G_IO_ERROR, G_IO_ERROR_NOT_INITIALIZED, _("Invalid object, not initialized"));
if (freetext)
{
g_free (text);
}
return -1;
}
/* If we reached here is because we need to convert the text so, we
convert it with the charset converter */
conv_text = g_convert_with_iconv (text,
len,
ostream->priv->iconv,
&conv_read,
&conv_written,
&err);
if (freetext)
{
g_free (text);
}
if (err != NULL)
{
remainder = len - conv_read;
/* Store the partial char for the next conversion */
if (err->code == G_CONVERT_ERROR_ILLEGAL_SEQUENCE &&
remainder < MAX_UNICHAR_LEN &&
(g_utf8_get_char_validated (text + conv_read, remainder) == (gunichar)-2))
{
ostream->priv->buffer = g_strndup (text + conv_read, remainder);
ostream->priv->buflen = remainder;
}
else
{
/* Something went wrong with the conversion,
propagate the error and finish */
g_propagate_error (error, err);
g_free (conv_text);
return -1;
}
}
text = conv_text;
len = conv_written;
freetext = TRUE;
}
/* validate */
valid = g_utf8_validate (text, len, &end);
nvalid = end - text;
/* Avoid keeping a CRLF across two buffers. */
if (valid && len > 1 && end[-1] == '\r')
@ -334,8 +707,8 @@ xed_document_output_stream_write (GOutputStream *stream,
if (!valid)
{
gsize nvalid = end - text;
gsize remainder = len - nvalid;
// gsize nvalid = end - text;
remainder = len - nvalid;
gunichar ch;
if ((remainder < MAX_UNICHAR_LEN) &&

8
xed/xed-document-output-stream.h
View File

@ -26,6 +26,7 @@
#include <gio/gio.h>
#include "xed-document.h"
#include "xed-encodings.h"
G_BEGIN_DECLS
@ -55,10 +56,15 @@ struct _XedDocumentOutputStreamClass
GType xed_document_output_stream_get_type (void) G_GNUC_CONST;
GOutputStream *xed_document_output_stream_new (XedDocument *doc);
GOutputStream *xed_document_output_stream_new (XedDocument *doc,
GSList *candidate_encodings);
XedDocumentNewlineType xed_document_output_stream_detect_newline_type (XedDocumentOutputStream *stream);
const XedEncoding *xed_document_output_stream_get_guessed (XedDocumentOutputStream *stream);
guint xed_document_output_stream_get_num_fallbacks (XedDocumentOutputStream *stream);
G_END_DECLS
#endif /* __XED_DOCUMENT_OUTPUT_STREAM_H__ */

422
xed/xed-smart-charset-converter.c
View File

@ -1,422 +0,0 @@
/*
* xed-smart-charset-converter.c
* This file is part of xed
*
* Copyright (C) 2009 - Ignacio Casal Quinteiro
*
* xed is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* xed is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with xed; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301 USA
*/
#include "xed-smart-charset-converter.h"
#include "xed-debug.h"
#include "xed-document.h"
#include <gio/gio.h>
#include <glib/gi18n.h>
#define XED_SMART_CHARSET_CONVERTER_GET_PRIVATE(object)(G_TYPE_INSTANCE_GET_PRIVATE((object), XED_TYPE_SMART_CHARSET_CONVERTER, XedSmartCharsetConverterPrivate))
struct _XedSmartCharsetConverterPrivate
{
GCharsetConverter *charset_conv;
GSList *encodings;
GSList *current_encoding;
guint is_utf8 : 1;
guint use_first : 1;
};
static void xed_smart_charset_converter_iface_init (GConverterIface *iface);
G_DEFINE_TYPE_WITH_CODE (XedSmartCharsetConverter, xed_smart_charset_converter,
G_TYPE_OBJECT,
G_IMPLEMENT_INTERFACE (G_TYPE_CONVERTER,
xed_smart_charset_converter_iface_init))
static void
xed_smart_charset_converter_finalize (GObject *object)
{
XedSmartCharsetConverter *smart = XED_SMART_CHARSET_CONVERTER (object);
g_slist_free (smart->priv->encodings);
xed_debug_message (DEBUG_UTILS, "finalizing smart charset converter");
G_OBJECT_CLASS (xed_smart_charset_converter_parent_class)->finalize (object);
}
static void
xed_smart_charset_converter_dispose (GObject *object)
{
XedSmartCharsetConverter *smart = XED_SMART_CHARSET_CONVERTER (object);
if (smart->priv->charset_conv != NULL)
{
g_object_unref (smart->priv->charset_conv);
smart->priv->charset_conv = NULL;
}
xed_debug_message (DEBUG_UTILS, "disposing smart charset converter");
G_OBJECT_CLASS (xed_smart_charset_converter_parent_class)->dispose (object);
}
static void
xed_smart_charset_converter_class_init (XedSmartCharsetConverterClass *klass)
{
GObjectClass *object_class = G_OBJECT_CLASS (klass);
object_class->finalize = xed_smart_charset_converter_finalize;
object_class->dispose = xed_smart_charset_converter_dispose;
g_type_class_add_private (object_class, sizeof (XedSmartCharsetConverterPrivate));
}
static void
xed_smart_charset_converter_init (XedSmartCharsetConverter *smart)
{
smart->priv = XED_SMART_CHARSET_CONVERTER_GET_PRIVATE (smart);
smart->priv->charset_conv = NULL;
smart->priv->encodings = NULL;
smart->priv->current_encoding = NULL;
smart->priv->is_utf8 = FALSE;
smart->priv->use_first = FALSE;
xed_debug_message (DEBUG_UTILS, "initializing smart charset converter");
}
static const XedEncoding *
get_encoding (XedSmartCharsetConverter *smart)
{
if (smart->priv->current_encoding == NULL)
{
smart->priv->current_encoding = smart->priv->encodings;
}
else
{
smart->priv->current_encoding = g_slist_next (smart->priv->current_encoding);
}
if (smart->priv->current_encoding != NULL)
return (const XedEncoding *)smart->priv->current_encoding->data;
#if 0
FIXME: uncomment this when using fallback
/* If we tried all encodings, we return the first encoding */
smart->priv->use_first = TRUE;
smart->priv->current_encoding = smart->priv->encodings;
return (const XedEncoding *)smart->priv->current_encoding->data;
#endif
return NULL;
}
static gboolean
try_convert (GCharsetConverter *converter,
const void *inbuf,
gsize inbuf_size)
{
GError *err;
gsize bytes_read, nread;
gsize bytes_written, nwritten;
GConverterResult res;
gchar *out;
gboolean ret;
gsize out_size;
if (inbuf == NULL || inbuf_size == 0)
{
return FALSE;
}
err = NULL;
nread = 0;
nwritten = 0;
out_size = inbuf_size * 4;
out = g_malloc (out_size);
do
{
res = g_converter_convert (G_CONVERTER (converter),
(void *) ((gsize) inbuf + nread),
inbuf_size - nread,
out + nwritten,
out_size - nwritten,
G_CONVERTER_INPUT_AT_END,
&bytes_read,
&bytes_written,
&err);
nread += bytes_read;
nwritten += bytes_written;
} while (res != G_CONVERTER_FINISHED && res != G_CONVERTER_ERROR && err == NULL);
if (err != NULL)
{
if (err->code == G_CONVERT_ERROR_PARTIAL_INPUT)
{
/* FIXME We can get partial input while guessing the
encoding because we just take some amount of text
to guess from. */
ret = TRUE;
}
else
{
ret = FALSE;
}
g_error_free (err);
}
else
{
ret = TRUE;
}
/* FIXME: Check the remainder? */
if (ret == TRUE && !g_utf8_validate (out, nwritten, NULL))
{
ret = FALSE;
}
g_free (out);
return ret;
}
static GCharsetConverter *
guess_encoding (XedSmartCharsetConverter *smart,
const void *inbuf,
gsize inbuf_size)
{
GCharsetConverter *conv = NULL;
if (inbuf == NULL || inbuf_size == 0)
{
smart->priv->is_utf8 = TRUE;
return NULL;
}
if (smart->priv->encodings != NULL &&
smart->priv->encodings->next == NULL)
smart->priv->use_first = TRUE;
/* We just check the first block */
while (TRUE)
{
const XedEncoding *enc;
if (conv != NULL)
{
g_object_unref (conv);
conv = NULL;
}
/* We get an encoding from the list */
enc = get_encoding (smart);
/* if it is NULL we didn't guess anything */
if (enc == NULL)
{
break;
}
xed_debug_message (DEBUG_UTILS, "trying charset: %s",
xed_encoding_get_charset (smart->priv->current_encoding->data));
if (enc == xed_encoding_get_utf8 ())
{
gsize remainder;
const gchar *end;
if (g_utf8_validate (inbuf, inbuf_size, &end) ||
smart->priv->use_first)
{
smart->priv->is_utf8 = TRUE;
break;
}
/* Check if the end is less than one char */
remainder = inbuf_size - (end - (gchar *)inbuf);
if (remainder < 6)
{
smart->priv->is_utf8 = TRUE;
break;
}
continue;
}
conv = g_charset_converter_new ("UTF-8",
xed_encoding_get_charset (enc),
NULL);
/* If we tried all encodings we use the first one */
if (smart->priv->use_first)
{
break;
}
/* Try to convert */
if (try_convert (conv, inbuf, inbuf_size))
{
break;
}
}
if (conv != NULL)
{
g_converter_reset (G_CONVERTER (conv));
/* FIXME: uncomment this when we want to use the fallback
g_charset_converter_set_use_fallback (conv, TRUE);*/
}
return conv;
}
static GConverterResult
xed_smart_charset_converter_convert (GConverter *converter,
const void *inbuf,
gsize inbuf_size,
void *outbuf,
gsize outbuf_size,
GConverterFlags flags,
gsize *bytes_read,
gsize *bytes_written,
GError **error)
{
XedSmartCharsetConverter *smart = XED_SMART_CHARSET_CONVERTER (converter);
/* Guess the encoding if we didn't make it yet */
if (smart->priv->charset_conv == NULL &&
!smart->priv->is_utf8)
{
smart->priv->charset_conv = guess_encoding (smart, inbuf, inbuf_size);
/* If we still have the previous case is that we didn't guess
anything */
if (smart->priv->charset_conv == NULL &&
!smart->priv->is_utf8)
{
/* FIXME: Add a different domain when we kill xed_convert */
g_set_error_literal (error, XED_DOCUMENT_ERROR,
XED_DOCUMENT_ERROR_ENCODING_AUTO_DETECTION_FAILED,
_("It is not possible to detect the encoding automatically"));
return G_CONVERTER_ERROR;
}
}
/* Now if the encoding is utf8 just redirect the input to the output */
if (smart->priv->is_utf8)
{
gsize size;
GConverterResult ret;
size = MIN (inbuf_size, outbuf_size);
memcpy (outbuf, inbuf, size);
*bytes_read = size;
*bytes_written = size;
ret = G_CONVERTER_CONVERTED;
if (flags & G_CONVERTER_INPUT_AT_END)
ret = G_CONVERTER_FINISHED;
else if (flags & G_CONVERTER_FLUSH)
ret = G_CONVERTER_FLUSHED;
return ret;
}
/* If we reached here is because we need to convert the text so, we
convert it with the charset converter */
return g_converter_convert (G_CONVERTER (smart->priv->charset_conv),
inbuf,
inbuf_size,
outbuf,
outbuf_size,
flags,
bytes_read,
bytes_written,
error);
}
static void
xed_smart_charset_converter_reset (GConverter *converter)
{
XedSmartCharsetConverter *smart = XED_SMART_CHARSET_CONVERTER (converter);
smart->priv->current_encoding = NULL;
smart->priv->is_utf8 = FALSE;
if (smart->priv->charset_conv != NULL)
{
g_object_unref (smart->priv->charset_conv);
smart->priv->charset_conv = NULL;
}
}
static void
xed_smart_charset_converter_iface_init (GConverterIface *iface)
{
iface->convert = xed_smart_charset_converter_convert;
iface->reset = xed_smart_charset_converter_reset;
}
XedSmartCharsetConverter *
xed_smart_charset_converter_new (GSList *candidate_encodings)
{
XedSmartCharsetConverter *smart;
g_return_val_if_fail (candidate_encodings != NULL, NULL);
smart = g_object_new (XED_TYPE_SMART_CHARSET_CONVERTER, NULL);
smart->priv->encodings = g_slist_copy (candidate_encodings);
return smart;
}
const XedEncoding *
xed_smart_charset_converter_get_guessed (XedSmartCharsetConverter *smart)
{
g_return_val_if_fail (XED_IS_SMART_CHARSET_CONVERTER (smart), NULL);
if (smart->priv->current_encoding != NULL)
{
return (const XedEncoding *)smart->priv->current_encoding->data;
}
else if (smart->priv->is_utf8)
{
return xed_encoding_get_utf8 ();
}
return NULL;
}
guint
xed_smart_charset_converter_get_num_fallbacks (XedSmartCharsetConverter *smart)
{
g_return_val_if_fail (XED_IS_SMART_CHARSET_CONVERTER (smart), FALSE);
if (smart->priv->charset_conv == NULL)
return FALSE;
return g_charset_converter_get_num_fallbacks (smart->priv->charset_conv) != 0;
}

66
xed/xed-smart-charset-converter.h
View File

@ -1,66 +0,0 @@
/*
* xed-smart-charset-converter.h
* This file is part of xed
*
* Copyright (C) 2009 - Ignacio Casal Quinteiro
*
* xed is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* xed is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with xed; if not, write to the Free Software
* Foundation, Inc., 51 Franklin St, Fifth Floor,
* Boston, MA 02110-1301 USA
*/
#ifndef __XED_SMART_CHARSET_CONVERTER_H__
#define __XED_SMART_CHARSET_CONVERTER_H__
#include <glib-object.h>
#include "xed-encodings.h"
G_BEGIN_DECLS
#define XED_TYPE_SMART_CHARSET_CONVERTER (xed_smart_charset_converter_get_type ())
#define XED_SMART_CHARSET_CONVERTER(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), XED_TYPE_SMART_CHARSET_CONVERTER, XedSmartCharsetConverter))
#define XED_SMART_CHARSET_CONVERTER_CONST(obj) (G_TYPE_CHECK_INSTANCE_CAST ((obj), XED_TYPE_SMART_CHARSET_CONVERTER, XedSmartCharsetConverter const))
#define XED_SMART_CHARSET_CONVERTER_CLASS(klass) (G_TYPE_CHECK_CLASS_CAST ((klass), XED_TYPE_SMART_CHARSET_CONVERTER, XedSmartCharsetConverterClass))
#define XED_IS_SMART_CHARSET_CONVERTER(obj) (G_TYPE_CHECK_INSTANCE_TYPE ((obj), XED_TYPE_SMART_CHARSET_CONVERTER))
#define XED_IS_SMART_CHARSET_CONVERTER_CLASS(klass) (G_TYPE_CHECK_CLASS_TYPE ((klass), XED_TYPE_SMART_CHARSET_CONVERTER))
#define XED_SMART_CHARSET_CONVERTER_GET_CLASS(obj) (G_TYPE_INSTANCE_GET_CLASS ((obj), XED_TYPE_SMART_CHARSET_CONVERTER, XedSmartCharsetConverterClass))
typedef struct _XedSmartCharsetConverter XedSmartCharsetConverter;
typedef struct _XedSmartCharsetConverterClass XedSmartCharsetConverterClass;
typedef struct _XedSmartCharsetConverterPrivate XedSmartCharsetConverterPrivate;
struct _XedSmartCharsetConverter
{
GObject parent;
XedSmartCharsetConverterPrivate *priv;
};
struct _XedSmartCharsetConverterClass
{
GObjectClass parent_class;
};
GType xed_smart_charset_converter_get_type (void) G_GNUC_CONST;
XedSmartCharsetConverter *xed_smart_charset_converter_new (GSList *candidate_encodings);
const XedEncoding *xed_smart_charset_converter_get_guessed (XedSmartCharsetConverter *smart);
guint xed_smart_charset_converter_get_num_fallbacks(XedSmartCharsetConverter *smart);
G_END_DECLS
#endif /* __XED_SMART_CHARSET_CONVERTER_H__ */