From 0f3ab84a261292d16f684551e67f2f007199936a Mon Sep 17 00:00:00 2001
From: Mark Pulford <mark@kyne.com.au>
Date: Wed, 5 Oct 2011 23:30:27 +1030
Subject: Support locales which use comma decimal separators
Some locales (cs_CZ, de_DE,..) use a comma as their decimal separator.
This causes CJSON to generate incorrect JSON (Eg, [10,1]), and fail when
parsing some valid JSON (Eg, [10,"test"]).
Added USE_POSIX_LOCALE #define which harnesses the thread-safe
POSIX.1-2008 locale support (newlocale(), uselocale(), freelocale())
to temporarily use the POSIX locale during JSON conversion.
Some older POSIX operating systems with xlocale.h (MacOSX) are also
supported.
---
Makefile | 10 +++++++---
NEWS | 3 +++
TODO | 2 ++
lua_cjson.c | 51 +++++++++++++++++++++++++++++++++++++++++++++++++++
tests/test.lua | 16 ++++++++++++++++
5 files changed, 79 insertions(+), 3 deletions(-)
diff --git a/Makefile b/Makefile
index d34ff6d..5731440 100644
--- a/Makefile
+++ b/Makefile
@@ -15,13 +15,17 @@ LDFLAGS += -shared
LUA_INCLUDE_DIR ?= $(PREFIX)/include
LUA_LIB_DIR ?= $(PREFIX)/lib/lua/$(LUA_VERSION)
-# Some versions of Solaris are missing isinf(). Add -DMISSING_ISINF to
-# CFLAGS to work around this bug.
-
#CFLAGS ?= -g -Wall -pedantic -fno-inline
CFLAGS ?= -g -O3 -Wall -pedantic
override CFLAGS += -fpic -I$(LUA_INCLUDE_DIR) -DVERSION=\"$(CJSON_VERSION)\"
+## Conditional work arounds
+# Handle Solaris platforms that are missing isinf().
+#override CFLAGS += -DMISSING_ISINF
+# Handle locales that use comma as a decimal separator on locale aware
+# platforms. Requires POSIX-1.2008 support.
+override CFLAGS += -DUSE_POSIX_LOCALE
+
INSTALL ?= install
.PHONY: all clean install package
diff --git a/NEWS b/NEWS
index 3527b04..634d9af 100644
--- a/NEWS
+++ b/NEWS
@@ -1,3 +1,6 @@
+Version 1.0.4 (?)
+* Handle locales with a comma decimal separator
+
Version 1.0.3 (Sep 15 2011)
* Fixed detection of objects with numeric string keys
* Provided work around for missing isinf() on Solaris
diff --git a/TODO b/TODO
index 1345448..9dbde9c 100644
--- a/TODO
+++ b/TODO
@@ -2,3 +2,5 @@
- Optionally create an object for settings. Clone function.
- Convert documentation into structured source format
+
+- Add setlocale() support for non-POSIX 2008 operating systems
diff --git a/lua_cjson.c b/lua_cjson.c
index f765883..151fa39 100644
--- a/lua_cjson.c
+++ b/lua_cjson.c
@@ -44,7 +44,30 @@
#include "strbuf.h"
+#ifdef USE_POSIX_LOCALE
+/* Reset locale to POSIX for strtod() / sprintf().
+ * Some locales use comma as a decimal separator. This breaks JSON. */
+
+/* unistd.h defines _POSIX_VERSION */
+#include <unistd.h>
+#if _POSIX_VERSION >= 200809L
+/* POSIX.1-2008 adds threadsafe locale support */
+#include <locale.h>
+#elif defined(_POSIX_VERSION)
+/* Some pre-POSIX.1-2008 operating systems use xlocale.h instead */
+#include <xlocale.h>
+#else
+#error Missing _POSIX_VERSION define
+#endif
+#define LOCALE_SET_POSIX(x) (x)->saved_locale = uselocale((x)->posix_locale)
+#define LOCALE_RESTORE(x) uselocale((x)->saved_locale)
+#else
+#define LOCALE_SET_POSIX(x) do { } while(0)
+#define LOCALE_RESTORE(x) do { } while(0)
+#endif
+
#ifdef MISSING_ISINF
+/* Some Solaris platforms are missing isinf(). Define here. */
#define isinf(x) (!isnan(x) && isnan((x) - (x)))
#endif
@@ -99,6 +122,10 @@ typedef struct {
char *char2escape[256]; /* Encoding */
#endif
strbuf_t encode_buf;
+#if USE_POSIX_LOCALE
+ locale_t saved_locale;
+ locale_t posix_locale;
+#endif
char number_fmt[8]; /* "%.XXg\0" */
int current_depth;
@@ -342,6 +369,10 @@ static int json_destroy_config(lua_State *l)
json_config_t *cfg;
cfg = lua_touserdata(l, 1);
+#ifdef USE_POSIX_LOCALE
+ if (cfg->posix_locale)
+ freelocale(cfg->posix_locale);
+#endif
if (cfg)
strbuf_free(&cfg->encode_buf);
cfg = NULL;
@@ -363,6 +394,13 @@ static void json_create_config(lua_State *l)
lua_setmetatable(l, -2);
strbuf_init(&cfg->encode_buf, 0);
+#if USE_POSIX_LOCALE
+ cfg->saved_locale = NULL;
+ /* Must not lua_error() before cfg->posix_locale has been initialised */
+ cfg->posix_locale = newlocale(LC_ALL_MASK, "C", NULL);
+ if (!cfg->posix_locale)
+ luaL_error(l, "Failed to create POSIX locale for JSON");
+#endif
cfg->encode_sparse_convert = DEFAULT_SPARSE_CONVERT;
cfg->encode_sparse_ratio = DEFAULT_SPARSE_RATIO;
@@ -452,6 +490,7 @@ static void json_encode_exception(lua_State *l, json_config_t *cfg, int lindex,
{
if (!cfg->encode_keep_buffer)
strbuf_free(&cfg->encode_buf);
+ LOCALE_RESTORE(cfg);
luaL_error(l, "Cannot serialise %s: %s",
lua_typename(l, lua_type(l, lindex)), reason);
}
@@ -542,6 +581,7 @@ static void json_encode_descend(lua_State *l, json_config_t *cfg)
if (cfg->current_depth > cfg->encode_max_depth) {
if (!cfg->encode_keep_buffer)
strbuf_free(&cfg->encode_buf);
+ LOCALE_RESTORE(cfg);
luaL_error(l, "Cannot serialise, excessive nesting (%d)",
cfg->current_depth);
}
@@ -701,9 +741,13 @@ static int json_encode(lua_State *l)
else
strbuf_init(&cfg->encode_buf, 0);
+ LOCALE_SET_POSIX(cfg);
+
json_append_data(l, cfg, &cfg->encode_buf);
json = strbuf_string(&cfg->encode_buf, &len);
+ LOCALE_RESTORE(cfg);
+
lua_pushlstring(l, json, len);
if (!cfg->encode_keep_buffer)
@@ -1084,6 +1128,8 @@ static void json_throw_parse_error(lua_State *l, json_parse_t *json,
else
found = json_token_type_name[token->type];
+ LOCALE_RESTORE(json->cfg);
+
/* Note: token->index is 0 based, display starting from 1 */
luaL_error(l, "Expected %s but found %s at character %d",
exp, found, token->index + 1);
@@ -1095,6 +1141,7 @@ static void json_decode_checkstack(lua_State *l, json_parse_t *json, int n)
return;
strbuf_free(json->tmp);
+ LOCALE_RESTORE(json->cfg);
luaL_error(l, "Too many nested data structures");
}
@@ -1224,6 +1271,8 @@ static void lua_json_decode(lua_State *l, const char *json_text, int json_len)
* string must be smaller than the entire json string */
json.tmp = strbuf_new(json_len);
+ LOCALE_SET_POSIX(json.cfg);
+
json_next_token(&json, &token);
json_process_value(l, &json, &token);
@@ -1233,6 +1282,8 @@ static void lua_json_decode(lua_State *l, const char *json_text, int json_len)
if (token.type != T_END)
json_throw_parse_error(l, &json, "the end", &token);
+ LOCALE_RESTORE(json.cfg);
+
strbuf_free(json.tmp);
}
diff --git a/tests/test.lua b/tests/test.lua
index 7a75243..d80dcf0 100755
--- a/tests/test.lua
+++ b/tests/test.lua
@@ -210,6 +210,21 @@ local escape_tests = {
{ json.decode, { utf16_escaped }, true, { utf8_raw } }
}
+-- The standard Lua interpreter is ANSI C online doesn't support locales
+-- by default. Force a known problematic locale to test strtod()/sprintf().
+local locale_tests = {
+ function ()
+ os.setlocale("cs_CZ")
+ return "Setting locale to cs_CZ (comma separator)"
+ end,
+ { json.encode, { 1.5 }, true, { '1.5' } },
+ { json.decode, { "[ 10, \"test\" ]" }, true, { { 10, "test" } } },
+ function ()
+ os.setlocale("C")
+ return "Reverting locale to POSIX"
+ end
+}
+
print(string.format("Testing CJSON v%s\n", cjson.version))
run_test_group("decode simple value", decode_simple_tests)
@@ -225,6 +240,7 @@ run_test_group("encode table", encode_table_tests)
run_test_group("decode error", decode_error_tests)
run_test_group("encode error", encode_error_tests)
run_test_group("escape", escape_tests)
+run_test_group("locale", locale_tests)
cjson.refuse_invalid_numbers(false)
cjson.encode_max_depth(20)
--
cgit v1.2.3-55-g6feb