| 1 | =head1 NAME | ||
| 2 | |||
| 3 | JSON::XS - JSON serialising/deserialising, done correctly and fast | ||
| 4 | |||
| 5 | =encoding utf-8 | ||
| 6 | |||
| 7 | JSON::XS - 正しくて高速な JSON シリアライザ/デシリアライザ | ||
| 8 | (http://fleur.hio.jp/perldoc/mix/lib/JSON/XS.html) | ||
| 9 | |||
| 10 | =head1 SYNOPSIS | ||
| 11 | |||
| 12 | use JSON::XS; | ||
| 13 | |||
| 14 | # exported functions, they croak on error | ||
| 15 | # and expect/generate UTF-8 | ||
| 16 | |||
| 17 | $utf8_encoded_json_text = encode_json $perl_hash_or_arrayref; | ||
| 18 | $perl_hash_or_arrayref = decode_json $utf8_encoded_json_text; | ||
| 19 | |||
| 20 | # OO-interface | ||
| 21 | |||
| 22 | $coder = JSON::XS->new->ascii->pretty->allow_nonref; | ||
| 23 | $pretty_printed_unencoded = $coder->encode ($perl_scalar); | ||
| 24 | $perl_scalar = $coder->decode ($unicode_json_text); | ||
| 25 | |||
| 26 | # Note that JSON version 2.0 and above will automatically use JSON::XS | ||
| 27 | # if available, at virtually no speed overhead either, so you should | ||
| 28 | # be able to just: | ||
| 29 | |||
| 30 | use JSON; | ||
| 31 | |||
| 32 | # and do the same things, except that you have a pure-perl fallback now. | ||
| 33 | |||
| 34 | =head1 DESCRIPTION | ||
| 35 | |||
| 36 | This module converts Perl data structures to JSON and vice versa. Its | ||
| 37 | primary goal is to be I<correct> and its secondary goal is to be | ||
| 38 | I<fast>. To reach the latter goal it was written in C. | ||
| 39 | |||
| 40 | See MAPPING, below, on how JSON::XS maps perl values to JSON values and | ||
| 41 | vice versa. | ||
| 42 | |||
| 43 | =head2 FEATURES | ||
| 44 | |||
| 45 | =over | ||
| 46 | |||
| 47 | =item * correct Unicode handling | ||
| 48 | |||
| 49 | This module knows how to handle Unicode, documents how and when it does | ||
| 50 | so, and even documents what "correct" means. | ||
| 51 | |||
| 52 | =item * round-trip integrity | ||
| 53 | |||
| 54 | When you serialise a perl data structure using only data types supported | ||
| 55 | by JSON and Perl, the deserialised data structure is identical on the Perl | ||
| 56 | level. (e.g. the string "2.0" doesn't suddenly become "2" just because | ||
| 57 | it looks like a number). There I<are> minor exceptions to this, read the | ||
| 58 | MAPPING section below to learn about those. | ||
| 59 | |||
| 60 | =item * strict checking of JSON correctness | ||
| 61 | |||
| 62 | There is no guessing, no generating of illegal JSON texts by default, | ||
| 63 | and only JSON is accepted as input by default (the latter is a security | ||
| 64 | feature). | ||
| 65 | |||
| 66 | =item * fast | ||
| 67 | |||
| 68 | Compared to other JSON modules and other serialisers such as Storable, | ||
| 69 | this module usually compares favourably in terms of speed, too. | ||
| 70 | |||
| 71 | =item * simple to use | ||
| 72 | |||
| 73 | This module has both a simple functional interface as well as an object | ||
| 74 | oriented interface. | ||
| 75 | |||
| 76 | =item * reasonably versatile output formats | ||
| 77 | |||
| 78 | You can choose between the most compact guaranteed-single-line format | ||
| 79 | possible (nice for simple line-based protocols), a pure-ASCII format | ||
| 80 | (for when your transport is not 8-bit clean, still supports the whole | ||
| 81 | Unicode range), or a pretty-printed format (for when you want to read that | ||
| 82 | stuff). Or you can combine those features in whatever way you like. | ||
| 83 | |||
| 84 | =back | ||
| 85 | |||
| 86 | =cut | ||
| 87 | |||
| 88 | package JSON::XS; | ||
| 89 | |||
| 90 | 25 | ▶ | use common::sense; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 91 | |||
| 92 | 25 | our $VERSION = '4.05'; | |
| 93 | 25 | our @ISA = qw(Exporter); | |
| 94 | |||
| 95 | 25 | our @EXPORT = qw(encode_json decode_json); | |
| 96 | |||
| 97 | 25 | ▶ | use Exporter; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 98 | 25 | ▶ | use XSLoader; |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 99 | |||
| 100 | 25 | ▶ | use Types::Serialiser (); |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 101 | |||
| 102 | =head1 FUNCTIONAL INTERFACE | ||
| 103 | |||
| 104 | The following convenience methods are provided by this module. They are | ||
| 105 | exported by default: | ||
| 106 | |||
| 107 | =over | ||
| 108 | |||
| 109 | =item $json_text = encode_json $perl_scalar | ||
| 110 | |||
| 111 | Converts the given Perl data structure to a UTF-8 encoded, binary string | ||
| 112 | (that is, the string contains octets only). Croaks on error. | ||
| 113 | |||
| 114 | This function call is functionally identical to: | ||
| 115 | |||
| 116 | $json_text = JSON::XS->new->utf8->encode ($perl_scalar) | ||
| 117 | |||
| 118 | Except being faster. | ||
| 119 | |||
| 120 | =item $perl_scalar = decode_json $json_text | ||
| 121 | |||
| 122 | The opposite of C<encode_json>: expects a UTF-8 (binary) string and tries | ||
| 123 | to parse that as a UTF-8 encoded JSON text, returning the resulting | ||
| 124 | reference. Croaks on error. | ||
| 125 | |||
| 126 | This function call is functionally identical to: | ||
| 127 | |||
| 128 | $perl_scalar = JSON::XS->new->utf8->decode ($json_text) | ||
| 129 | |||
| 130 | Except being faster. | ||
| 131 | |||
| 132 | =back | ||
| 133 | |||
| 134 | |||
| 135 | =head1 A FEW NOTES ON UNICODE AND PERL | ||
| 136 | |||
| 137 | Since this often leads to confusion, here are a few very clear words on | ||
| 138 | how Unicode works in Perl, modulo bugs. | ||
| 139 | |||
| 140 | =over | ||
| 141 | |||
| 142 | =item 1. Perl strings can store characters with ordinal values > 255. | ||
| 143 | |||
| 144 | This enables you to store Unicode characters as single characters in a | ||
| 145 | Perl string - very natural. | ||
| 146 | |||
| 147 | =item 2. Perl does I<not> associate an encoding with your strings. | ||
| 148 | |||
| 149 | ... until you force it to, e.g. when matching it against a regex, or | ||
| 150 | printing the scalar to a file, in which case Perl either interprets your | ||
| 151 | string as locale-encoded text, octets/binary, or as Unicode, depending | ||
| 152 | on various settings. In no case is an encoding stored together with your | ||
| 153 | data, it is I<use> that decides encoding, not any magical meta data. | ||
| 154 | |||
| 155 | =item 3. The internal utf-8 flag has no meaning with regards to the | ||
| 156 | encoding of your string. | ||
| 157 | |||
| 158 | Just ignore that flag unless you debug a Perl bug, a module written in | ||
| 159 | XS or want to dive into the internals of perl. Otherwise it will only | ||
| 160 | confuse you, as, despite the name, it says nothing about how your string | ||
| 161 | is encoded. You can have Unicode strings with that flag set, with that | ||
| 162 | flag clear, and you can have binary data with that flag set and that flag | ||
| 163 | clear. Other possibilities exist, too. | ||
| 164 | |||
| 165 | If you didn't know about that flag, just the better, pretend it doesn't | ||
| 166 | exist. | ||
| 167 | |||
| 168 | =item 4. A "Unicode String" is simply a string where each character can be | ||
| 169 | validly interpreted as a Unicode code point. | ||
| 170 | |||
| 171 | If you have UTF-8 encoded data, it is no longer a Unicode string, but a | ||
| 172 | Unicode string encoded in UTF-8, giving you a binary string. | ||
| 173 | |||
| 174 | =item 5. A string containing "high" (> 255) character values is I<not> a UTF-8 string. | ||
| 175 | |||
| 176 | It's a fact. Learn to live with it. | ||
| 177 | |||
| 178 | =back | ||
| 179 | |||
| 180 | I hope this helps :) | ||
| 181 | |||
| 182 | |||
| 183 | =head1 OBJECT-ORIENTED INTERFACE | ||
| 184 | |||
| 185 | The object oriented interface lets you configure your own encoding or | ||
| 186 | decoding style, within the limits of supported formats. | ||
| 187 | |||
| 188 | =over | ||
| 189 | |||
| 190 | =item $json = new JSON::XS | ||
| 191 | |||
| 192 | Creates a new JSON::XS object that can be used to de/encode JSON | ||
| 193 | strings. All boolean flags described below are by default I<disabled> | ||
| 194 | (with the exception of C<allow_nonref>, which defaults to I<enabled> since | ||
| 195 | version C<4.0>). | ||
| 196 | |||
| 197 | The mutators for flags all return the JSON object again and thus calls can | ||
| 198 | be chained: | ||
| 199 | |||
| 200 | my $json = JSON::XS->new->utf8->space_after->encode ({a => [1,2]}) | ||
| 201 | => {"a": [1, 2]} | ||
| 202 | |||
| 203 | =item $json = $json->ascii ([$enable]) | ||
| 204 | |||
| 205 | =item $enabled = $json->get_ascii | ||
| 206 | |||
| 207 | If C<$enable> is true (or missing), then the C<encode> method will not | ||
| 208 | generate characters outside the code range C<0..127> (which is ASCII). Any | ||
| 209 | Unicode characters outside that range will be escaped using either a | ||
| 210 | single \uXXXX (BMP characters) or a double \uHHHH\uLLLLL escape sequence, | ||
| 211 | as per RFC4627. The resulting encoded JSON text can be treated as a native | ||
| 212 | Unicode string, an ascii-encoded, latin1-encoded or UTF-8 encoded string, | ||
| 213 | or any other superset of ASCII. | ||
| 214 | |||
| 215 | If C<$enable> is false, then the C<encode> method will not escape Unicode | ||
| 216 | characters unless required by the JSON syntax or other flags. This results | ||
| 217 | in a faster and more compact format. | ||
| 218 | |||
| 219 | See also the section I<ENCODING/CODESET FLAG NOTES> later in this | ||
| 220 | document. | ||
| 221 | |||
| 222 | The main use for this flag is to produce JSON texts that can be | ||
| 223 | transmitted over a 7-bit channel, as the encoded JSON texts will not | ||
| 224 | contain any 8 bit characters. | ||
| 225 | |||
| 226 | JSON::XS->new->ascii (1)->encode ([chr 0x10401]) | ||
| 227 | => ["\ud801\udc01"] | ||
| 228 | |||
| 229 | =item $json = $json->latin1 ([$enable]) | ||
| 230 | |||
| 231 | =item $enabled = $json->get_latin1 | ||
| 232 | |||
| 233 | If C<$enable> is true (or missing), then the C<encode> method will encode | ||
| 234 | the resulting JSON text as latin1 (or iso-8859-1), escaping any characters | ||
| 235 | outside the code range C<0..255>. The resulting string can be treated as a | ||
| 236 | latin1-encoded JSON text or a native Unicode string. The C<decode> method | ||
| 237 | will not be affected in any way by this flag, as C<decode> by default | ||
| 238 | expects Unicode, which is a strict superset of latin1. | ||
| 239 | |||
| 240 | If C<$enable> is false, then the C<encode> method will not escape Unicode | ||
| 241 | characters unless required by the JSON syntax or other flags. | ||
| 242 | |||
| 243 | See also the section I<ENCODING/CODESET FLAG NOTES> later in this | ||
| 244 | document. | ||
| 245 | |||
| 246 | The main use for this flag is efficiently encoding binary data as JSON | ||
| 247 | text, as most octets will not be escaped, resulting in a smaller encoded | ||
| 248 | size. The disadvantage is that the resulting JSON text is encoded | ||
| 249 | in latin1 (and must correctly be treated as such when storing and | ||
| 250 | transferring), a rare encoding for JSON. It is therefore most useful when | ||
| 251 | you want to store data structures known to contain binary data efficiently | ||
| 252 | in files or databases, not when talking to other JSON encoders/decoders. | ||
| 253 | |||
| 254 | JSON::XS->new->latin1->encode (["\x{89}\x{abc}"] | ||
| 255 | => ["\x{89}\\u0abc"] # (perl syntax, U+abc escaped, U+89 not) | ||
| 256 | |||
| 257 | =item $json = $json->utf8 ([$enable]) | ||
| 258 | |||
| 259 | =item $enabled = $json->get_utf8 | ||
| 260 | |||
| 261 | If C<$enable> is true (or missing), then the C<encode> method will encode | ||
| 262 | the JSON result into UTF-8, as required by many protocols, while the | ||
| 263 | C<decode> method expects to be handed a UTF-8-encoded string. Please | ||
| 264 | note that UTF-8-encoded strings do not contain any characters outside the | ||
| 265 | range C<0..255>, they are thus useful for bytewise/binary I/O. In future | ||
| 266 | versions, enabling this option might enable autodetection of the UTF-16 | ||
| 267 | and UTF-32 encoding families, as described in RFC4627. | ||
| 268 | |||
| 269 | If C<$enable> is false, then the C<encode> method will return the JSON | ||
| 270 | string as a (non-encoded) Unicode string, while C<decode> expects thus a | ||
| 271 | Unicode string. Any decoding or encoding (e.g. to UTF-8 or UTF-16) needs | ||
| 272 | to be done yourself, e.g. using the Encode module. | ||
| 273 | |||
| 274 | See also the section I<ENCODING/CODESET FLAG NOTES> later in this | ||
| 275 | document. | ||
| 276 | |||
| 277 | Example, output UTF-16BE-encoded JSON: | ||
| 278 | |||
| 279 | use Encode; | ||
| 280 | $jsontext = encode "UTF-16BE", JSON::XS->new->encode ($object); | ||
| 281 | |||
| 282 | Example, decode UTF-32LE-encoded JSON: | ||
| 283 | |||
| 284 | use Encode; | ||
| 285 | $object = JSON::XS->new->decode (decode "UTF-32LE", $jsontext); | ||
| 286 | |||
| 287 | =item $json = $json->pretty ([$enable]) | ||
| 288 | |||
| 289 | This enables (or disables) all of the C<indent>, C<space_before> and | ||
| 290 | C<space_after> (and in the future possibly more) flags in one call to | ||
| 291 | generate the most readable (or most compact) form possible. | ||
| 292 | |||
| 293 | Example, pretty-print some simple structure: | ||
| 294 | |||
| 295 | my $json = JSON::XS->new->pretty(1)->encode ({a => [1,2]}) | ||
| 296 | => | ||
| 297 | { | ||
| 298 | "a" : [ | ||
| 299 | 1, | ||
| 300 | 2 | ||
| 301 | ] | ||
| 302 | } | ||
| 303 | |||
| 304 | =item $json = $json->indent ([$enable]) | ||
| 305 | |||
| 306 | =item $enabled = $json->get_indent | ||
| 307 | |||
| 308 | If C<$enable> is true (or missing), then the C<encode> method will use a multiline | ||
| 309 | format as output, putting every array member or object/hash key-value pair | ||
| 310 | into its own line, indenting them properly. | ||
| 311 | |||
| 312 | If C<$enable> is false, no newlines or indenting will be produced, and the | ||
| 313 | resulting JSON text is guaranteed not to contain any C<newlines>. | ||
| 314 | |||
| 315 | This setting has no effect when decoding JSON texts. | ||
| 316 | |||
| 317 | =item $json = $json->space_before ([$enable]) | ||
| 318 | |||
| 319 | =item $enabled = $json->get_space_before | ||
| 320 | |||
| 321 | If C<$enable> is true (or missing), then the C<encode> method will add an extra | ||
| 322 | optional space before the C<:> separating keys from values in JSON objects. | ||
| 323 | |||
| 324 | If C<$enable> is false, then the C<encode> method will not add any extra | ||
| 325 | space at those places. | ||
| 326 | |||
| 327 | This setting has no effect when decoding JSON texts. You will also | ||
| 328 | most likely combine this setting with C<space_after>. | ||
| 329 | |||
| 330 | Example, space_before enabled, space_after and indent disabled: | ||
| 331 | |||
| 332 | {"key" :"value"} | ||
| 333 | |||
| 334 | =item $json = $json->space_after ([$enable]) | ||
| 335 | |||
| 336 | =item $enabled = $json->get_space_after | ||
| 337 | |||
| 338 | If C<$enable> is true (or missing), then the C<encode> method will add an extra | ||
| 339 | optional space after the C<:> separating keys from values in JSON objects | ||
| 340 | and extra whitespace after the C<,> separating key-value pairs and array | ||
| 341 | members. | ||
| 342 | |||
| 343 | If C<$enable> is false, then the C<encode> method will not add any extra | ||
| 344 | space at those places. | ||
| 345 | |||
| 346 | This setting has no effect when decoding JSON texts. | ||
| 347 | |||
| 348 | Example, space_before and indent disabled, space_after enabled: | ||
| 349 | |||
| 350 | {"key": "value"} | ||
| 351 | |||
| 352 | =item $json = $json->relaxed ([$enable]) | ||
| 353 | |||
| 354 | =item $enabled = $json->get_relaxed | ||
| 355 | |||
| 356 | If C<$enable> is true (or missing), then C<decode> will accept some | ||
| 357 | extensions to normal JSON syntax (see below). C<encode> will not be | ||
| 358 | affected in any way. I<Be aware that this option makes you accept invalid | ||
| 359 | JSON texts as if they were valid!>. I suggest only to use this option to | ||
| 360 | parse application-specific files written by humans (configuration files, | ||
| 361 | resource files etc.) | ||
| 362 | |||
| 363 | If C<$enable> is false (the default), then C<decode> will only accept | ||
| 364 | valid JSON texts. | ||
| 365 | |||
| 366 | Currently accepted extensions are: | ||
| 367 | |||
| 368 | =over | ||
| 369 | |||
| 370 | =item * list items can have an end-comma | ||
| 371 | |||
| 372 | JSON I<separates> array elements and key-value pairs with commas. This | ||
| 373 | can be annoying if you write JSON texts manually and want to be able to | ||
| 374 | quickly append elements, so this extension accepts comma at the end of | ||
| 375 | such items not just between them: | ||
| 376 | |||
| 377 | [ | ||
| 378 | 1, | ||
| 379 | 2, <- this comma not normally allowed | ||
| 380 | ] | ||
| 381 | { | ||
| 382 | "k1": "v1", | ||
| 383 | "k2": "v2", <- this comma not normally allowed | ||
| 384 | } | ||
| 385 | |||
| 386 | =item * shell-style '#'-comments | ||
| 387 | |||
| 388 | Whenever JSON allows whitespace, shell-style comments are additionally | ||
| 389 | allowed. They are terminated by the first carriage-return or line-feed | ||
| 390 | character, after which more white-space and comments are allowed. | ||
| 391 | |||
| 392 | [ | ||
| 393 | 1, # this comment not allowed in JSON | ||
| 394 | # neither this one... | ||
| 395 | ] | ||
| 396 | |||
| 397 | =item * literal ASCII TAB characters in strings | ||
| 398 | |||
| 399 | Literal ASCII TAB characters are now allowed in strings (and treated as | ||
| 400 | C<\t>). | ||
| 401 | |||
| 402 | [ | ||
| 403 | "Hello\tWorld", | ||
| 404 | "Hello<TAB>World", # literal <TAB> would not normally be allowed | ||
| 405 | ] | ||
| 406 | |||
| 407 | =back | ||
| 408 | |||
| 409 | =item $json = $json->canonical ([$enable]) | ||
| 410 | |||
| 411 | =item $enabled = $json->get_canonical | ||
| 412 | |||
| 413 | If C<$enable> is true (or missing), then the C<encode> method will output JSON objects | ||
| 414 | by sorting their keys. This is adding a comparatively high overhead. | ||
| 415 | |||
| 416 | If C<$enable> is false, then the C<encode> method will output key-value | ||
| 417 | pairs in the order Perl stores them (which will likely change between runs | ||
| 418 | of the same script, and can change even within the same run from 5.18 | ||
| 419 | onwards). | ||
| 420 | |||
| 421 | This option is useful if you want the same data structure to be encoded as | ||
| 422 | the same JSON text (given the same overall settings). If it is disabled, | ||
| 423 | the same hash might be encoded differently even if contains the same data, | ||
| 424 | as key-value pairs have no inherent ordering in Perl. | ||
| 425 | |||
| 426 | This setting has no effect when decoding JSON texts. | ||
| 427 | |||
| 428 | This setting has currently no effect on tied hashes. | ||
| 429 | |||
| 430 | =item $json = $json->allow_nonref ([$enable]) | ||
| 431 | |||
| 432 | =item $enabled = $json->get_allow_nonref | ||
| 433 | |||
| 434 | Unlike other boolean options, this option is enabled by default beginning | ||
| 435 | with version C<4.0>. See L<SECURITY CONSIDERATIONS> for the gory details. | ||
| 436 | |||
| 437 | If C<$enable> is true (or missing), then the C<encode> method can convert a | ||
| 438 | non-reference into its corresponding string, number or null JSON value, | ||
| 439 | which is an extension to RFC4627. Likewise, C<decode> will accept those JSON | ||
| 440 | values instead of croaking. | ||
| 441 | |||
| 442 | If C<$enable> is false, then the C<encode> method will croak if it isn't | ||
| 443 | passed an arrayref or hashref, as JSON texts must either be an object | ||
| 444 | or array. Likewise, C<decode> will croak if given something that is not a | ||
| 445 | JSON object or array. | ||
| 446 | |||
| 447 | Example, encode a Perl scalar as JSON value without enabled C<allow_nonref>, | ||
| 448 | resulting in an error: | ||
| 449 | |||
| 450 | JSON::XS->new->allow_nonref (0)->encode ("Hello, World!") | ||
| 451 | => hash- or arrayref expected... | ||
| 452 | |||
| 453 | =item $json = $json->allow_unknown ([$enable]) | ||
| 454 | |||
| 455 | =item $enabled = $json->get_allow_unknown | ||
| 456 | |||
| 457 | If C<$enable> is true (or missing), then C<encode> will I<not> throw an | ||
| 458 | exception when it encounters values it cannot represent in JSON (for | ||
| 459 | example, filehandles) but instead will encode a JSON C<null> value. Note | ||
| 460 | that blessed objects are not included here and are handled separately by | ||
| 461 | c<allow_nonref>. | ||
| 462 | |||
| 463 | If C<$enable> is false (the default), then C<encode> will throw an | ||
| 464 | exception when it encounters anything it cannot encode as JSON. | ||
| 465 | |||
| 466 | This option does not affect C<decode> in any way, and it is recommended to | ||
| 467 | leave it off unless you know your communications partner. | ||
| 468 | |||
| 469 | =item $json = $json->allow_blessed ([$enable]) | ||
| 470 | |||
| 471 | =item $enabled = $json->get_allow_blessed | ||
| 472 | |||
| 473 | See L<OBJECT SERIALISATION> for details. | ||
| 474 | |||
| 475 | If C<$enable> is true (or missing), then the C<encode> method will not | ||
| 476 | barf when it encounters a blessed reference that it cannot convert | ||
| 477 | otherwise. Instead, a JSON C<null> value is encoded instead of the object. | ||
| 478 | |||
| 479 | If C<$enable> is false (the default), then C<encode> will throw an | ||
| 480 | exception when it encounters a blessed object that it cannot convert | ||
| 481 | otherwise. | ||
| 482 | |||
| 483 | This setting has no effect on C<decode>. | ||
| 484 | |||
| 485 | =item $json = $json->convert_blessed ([$enable]) | ||
| 486 | |||
| 487 | =item $enabled = $json->get_convert_blessed | ||
| 488 | |||
| 489 | See L<OBJECT SERIALISATION> for details. | ||
| 490 | |||
| 491 | If C<$enable> is true (or missing), then C<encode>, upon encountering a | ||
| 492 | blessed object, will check for the availability of the C<TO_JSON> method | ||
| 493 | on the object's class. If found, it will be called in scalar context and | ||
| 494 | the resulting scalar will be encoded instead of the object. | ||
| 495 | |||
| 496 | The C<TO_JSON> method may safely call die if it wants. If C<TO_JSON> | ||
| 497 | returns other blessed objects, those will be handled in the same | ||
| 498 | way. C<TO_JSON> must take care of not causing an endless recursion cycle | ||
| 499 | (== crash) in this case. The name of C<TO_JSON> was chosen because other | ||
| 500 | methods called by the Perl core (== not by the user of the object) are | ||
| 501 | usually in upper case letters and to avoid collisions with any C<to_json> | ||
| 502 | function or method. | ||
| 503 | |||
| 504 | If C<$enable> is false (the default), then C<encode> will not consider | ||
| 505 | this type of conversion. | ||
| 506 | |||
| 507 | This setting has no effect on C<decode>. | ||
| 508 | |||
| 509 | =item $json = $json->allow_tags ([$enable]) | ||
| 510 | |||
| 511 | =item $enabled = $json->get_allow_tags | ||
| 512 | |||
| 513 | See L<OBJECT SERIALISATION> for details. | ||
| 514 | |||
| 515 | If C<$enable> is true (or missing), then C<encode>, upon encountering a | ||
| 516 | blessed object, will check for the availability of the C<FREEZE> method on | ||
| 517 | the object's class. If found, it will be used to serialise the object into | ||
| 518 | a nonstandard tagged JSON value (that JSON decoders cannot decode). | ||
| 519 | |||
| 520 | It also causes C<decode> to parse such tagged JSON values and deserialise | ||
| 521 | them via a call to the C<THAW> method. | ||
| 522 | |||
| 523 | If C<$enable> is false (the default), then C<encode> will not consider | ||
| 524 | this type of conversion, and tagged JSON values will cause a parse error | ||
| 525 | in C<decode>, as if tags were not part of the grammar. | ||
| 526 | |||
| 527 | =item $json->boolean_values ([$false, $true]) | ||
| 528 | |||
| 529 | =item ($false, $true) = $json->get_boolean_values | ||
| 530 | |||
| 531 | By default, JSON booleans will be decoded as overloaded | ||
| 532 | C<$Types::Serialiser::false> and C<$Types::Serialiser::true> objects. | ||
| 533 | |||
| 534 | With this method you can specify your own boolean values for decoding - | ||
| 535 | on decode, JSON C<false> will be decoded as a copy of C<$false>, and JSON | ||
| 536 | C<true> will be decoded as C<$true> ("copy" here is the same thing as | ||
| 537 | assigning a value to another variable, i.e. C<$copy = $false>). | ||
| 538 | |||
| 539 | Calling this method without any arguments will reset the booleans | ||
| 540 | to their default values. | ||
| 541 | |||
| 542 | C<get_boolean_values> will return both C<$false> and C<$true> values, or | ||
| 543 | the empty list when they are set to the default. | ||
| 544 | |||
| 545 | =item $json = $json->filter_json_object ([$coderef->($hashref)]) | ||
| 546 | |||
| 547 | When C<$coderef> is specified, it will be called from C<decode> each | ||
| 548 | time it decodes a JSON object. The only argument is a reference to | ||
| 549 | the newly-created hash. If the code reference returns a single scalar | ||
| 550 | (which need not be a reference), this value (or rather a copy of it) is | ||
| 551 | inserted into the deserialised data structure. If it returns an empty | ||
| 552 | list (NOTE: I<not> C<undef>, which is a valid scalar), the original | ||
| 553 | deserialised hash will be inserted. This setting can slow down decoding | ||
| 554 | considerably. | ||
| 555 | |||
| 556 | When C<$coderef> is omitted or undefined, any existing callback will | ||
| 557 | be removed and C<decode> will not change the deserialised hash in any | ||
| 558 | way. | ||
| 559 | |||
| 560 | Example, convert all JSON objects into the integer 5: | ||
| 561 | |||
| 562 | my $js = JSON::XS->new->filter_json_object (sub { 5 }); | ||
| 563 | # returns [5] | ||
| 564 | $js->decode ('[{}]') | ||
| 565 | # throw an exception because allow_nonref is not enabled | ||
| 566 | # so a lone 5 is not allowed. | ||
| 567 | $js->decode ('{"a":1, "b":2}'); | ||
| 568 | |||
| 569 | =item $json = $json->filter_json_single_key_object ($key [=> $coderef->($value)]) | ||
| 570 | |||
| 571 | Works remotely similar to C<filter_json_object>, but is only called for | ||
| 572 | JSON objects having a single key named C<$key>. | ||
| 573 | |||
| 574 | This C<$coderef> is called before the one specified via | ||
| 575 | C<filter_json_object>, if any. It gets passed the single value in the JSON | ||
| 576 | object. If it returns a single value, it will be inserted into the data | ||
| 577 | structure. If it returns nothing (not even C<undef> but the empty list), | ||
| 578 | the callback from C<filter_json_object> will be called next, as if no | ||
| 579 | single-key callback were specified. | ||
| 580 | |||
| 581 | If C<$coderef> is omitted or undefined, the corresponding callback will be | ||
| 582 | disabled. There can only ever be one callback for a given key. | ||
| 583 | |||
| 584 | As this callback gets called less often then the C<filter_json_object> | ||
| 585 | one, decoding speed will not usually suffer as much. Therefore, single-key | ||
| 586 | objects make excellent targets to serialise Perl objects into, especially | ||
| 587 | as single-key JSON objects are as close to the type-tagged value concept | ||
| 588 | as JSON gets (it's basically an ID/VALUE tuple). Of course, JSON does not | ||
| 589 | support this in any way, so you need to make sure your data never looks | ||
| 590 | like a serialised Perl hash. | ||
| 591 | |||
| 592 | Typical names for the single object key are C<__class_whatever__>, or | ||
| 593 | C<$__dollars_are_rarely_used__$> or C<}ugly_brace_placement>, or even | ||
| 594 | things like C<__class_md5sum(classname)__>, to reduce the risk of clashing | ||
| 595 | with real hashes. | ||
| 596 | |||
| 597 | Example, decode JSON objects of the form C<< { "__widget__" => <id> } >> | ||
| 598 | into the corresponding C<< $WIDGET{<id>} >> object: | ||
| 599 | |||
| 600 | # return whatever is in $WIDGET{5}: | ||
| 601 | JSON::XS | ||
| 602 | ->new | ||
| 603 | ->filter_json_single_key_object (__widget__ => sub { | ||
| 604 | $WIDGET{ $_[0] } | ||
| 605 | }) | ||
| 606 | ->decode ('{"__widget__": 5') | ||
| 607 | |||
| 608 | # this can be used with a TO_JSON method in some "widget" class | ||
| 609 | # for serialisation to json: | ||
| 610 | sub WidgetBase::TO_JSON { | ||
| 611 | my ($self) = @_; | ||
| 612 | |||
| 613 | unless ($self->{id}) { | ||
| 614 | $self->{id} = ..get..some..id..; | ||
| 615 | $WIDGET{$self->{id}} = $self; | ||
| 616 | } | ||
| 617 | |||
| 618 | { __widget__ => $self->{id} } | ||
| 619 | } | ||
| 620 | |||
| 621 | =item $json = $json->shrink ([$enable]) | ||
| 622 | |||
| 623 | =item $enabled = $json->get_shrink | ||
| 624 | |||
| 625 | Perl usually over-allocates memory a bit when allocating space for | ||
| 626 | strings. This flag optionally resizes strings generated by either | ||
| 627 | C<encode> or C<decode> to their minimum size possible. This can save | ||
| 628 | memory when your JSON texts are either very very long or you have many | ||
| 629 | short strings. It will also try to downgrade any strings to octet-form | ||
| 630 | if possible: perl stores strings internally either in an encoding called | ||
| 631 | UTF-X or in octet-form. The latter cannot store everything but uses less | ||
| 632 | space in general (and some buggy Perl or C code might even rely on that | ||
| 633 | internal representation being used). | ||
| 634 | |||
| 635 | The actual definition of what shrink does might change in future versions, | ||
| 636 | but it will always try to save space at the expense of time. | ||
| 637 | |||
| 638 | If C<$enable> is true (or missing), the string returned by C<encode> will | ||
| 639 | be shrunk-to-fit, while all strings generated by C<decode> will also be | ||
| 640 | shrunk-to-fit. | ||
| 641 | |||
| 642 | If C<$enable> is false, then the normal perl allocation algorithms are used. | ||
| 643 | If you work with your data, then this is likely to be faster. | ||
| 644 | |||
| 645 | In the future, this setting might control other things, such as converting | ||
| 646 | strings that look like integers or floats into integers or floats | ||
| 647 | internally (there is no difference on the Perl level), saving space. | ||
| 648 | |||
| 649 | =item $json = $json->max_depth ([$maximum_nesting_depth]) | ||
| 650 | |||
| 651 | =item $max_depth = $json->get_max_depth | ||
| 652 | |||
| 653 | Sets the maximum nesting level (default C<512>) accepted while encoding | ||
| 654 | or decoding. If a higher nesting level is detected in JSON text or a Perl | ||
| 655 | data structure, then the encoder and decoder will stop and croak at that | ||
| 656 | point. | ||
| 657 | |||
| 658 | Nesting level is defined by number of hash- or arrayrefs that the encoder | ||
| 659 | needs to traverse to reach a given point or the number of C<{> or C<[> | ||
| 660 | characters without their matching closing parenthesis crossed to reach a | ||
| 661 | given character in a string. | ||
| 662 | |||
| 663 | Setting the maximum depth to one disallows any nesting, so that ensures | ||
| 664 | that the object is only a single hash/object or array. | ||
| 665 | |||
| 666 | If no argument is given, the highest possible setting will be used, which | ||
| 667 | is rarely useful. | ||
| 668 | |||
| 669 | Note that nesting is implemented by recursion in C. The default value has | ||
| 670 | been chosen to be as large as typical operating systems allow without | ||
| 671 | crashing. | ||
| 672 | |||
| 673 | See SECURITY CONSIDERATIONS, below, for more info on why this is useful. | ||
| 674 | |||
| 675 | =item $json = $json->max_size ([$maximum_string_size]) | ||
| 676 | |||
| 677 | =item $max_size = $json->get_max_size | ||
| 678 | |||
| 679 | Set the maximum length a JSON text may have (in bytes) where decoding is | ||
| 680 | being attempted. The default is C<0>, meaning no limit. When C<decode> | ||
| 681 | is called on a string that is longer then this many bytes, it will not | ||
| 682 | attempt to decode the string but throw an exception. This setting has no | ||
| 683 | effect on C<encode> (yet). | ||
| 684 | |||
| 685 | If no argument is given, the limit check will be deactivated (same as when | ||
| 686 | C<0> is specified). | ||
| 687 | |||
| 688 | See SECURITY CONSIDERATIONS, below, for more info on why this is useful. | ||
| 689 | |||
| 690 | =item $json_text = $json->encode ($perl_scalar) | ||
| 691 | |||
| 692 | Converts the given Perl value or data structure to its JSON | ||
| 693 | representation. Croaks on error. | ||
| 694 | |||
| 695 | =item $perl_scalar = $json->decode ($json_text) | ||
| 696 | |||
| 697 | The opposite of C<encode>: expects a JSON text and tries to parse it, | ||
| 698 | returning the resulting simple scalar or reference. Croaks on error. | ||
| 699 | |||
| 700 | =item ($perl_scalar, $characters) = $json->decode_prefix ($json_text) | ||
| 701 | |||
| 702 | This works like the C<decode> method, but instead of raising an exception | ||
| 703 | when there is trailing garbage after the first JSON object, it will | ||
| 704 | silently stop parsing there and return the number of characters consumed | ||
| 705 | so far. | ||
| 706 | |||
| 707 | This is useful if your JSON texts are not delimited by an outer protocol | ||
| 708 | and you need to know where the JSON text ends. | ||
| 709 | |||
| 710 | JSON::XS->new->decode_prefix ("[1] the tail") | ||
| 711 | => ([1], 3) | ||
| 712 | |||
| 713 | =back | ||
| 714 | |||
| 715 | |||
| 716 | =head1 INCREMENTAL PARSING | ||
| 717 | |||
| 718 | In some cases, there is the need for incremental parsing of JSON | ||
| 719 | texts. While this module always has to keep both JSON text and resulting | ||
| 720 | Perl data structure in memory at one time, it does allow you to parse a | ||
| 721 | JSON stream incrementally. It does so by accumulating text until it has | ||
| 722 | a full JSON object, which it then can decode. This process is similar to | ||
| 723 | using C<decode_prefix> to see if a full JSON object is available, but | ||
| 724 | is much more efficient (and can be implemented with a minimum of method | ||
| 725 | calls). | ||
| 726 | |||
| 727 | JSON::XS will only attempt to parse the JSON text once it is sure it | ||
| 728 | has enough text to get a decisive result, using a very simple but | ||
| 729 | truly incremental parser. This means that it sometimes won't stop as | ||
| 730 | early as the full parser, for example, it doesn't detect mismatched | ||
| 731 | parentheses. The only thing it guarantees is that it starts decoding as | ||
| 732 | soon as a syntactically valid JSON text has been seen. This means you need | ||
| 733 | to set resource limits (e.g. C<max_size>) to ensure the parser will stop | ||
| 734 | parsing in the presence if syntax errors. | ||
| 735 | |||
| 736 | The following methods implement this incremental parser. | ||
| 737 | |||
| 738 | =over | ||
| 739 | |||
| 740 | =item [void, scalar or list context] = $json->incr_parse ([$string]) | ||
| 741 | |||
| 742 | This is the central parsing function. It can both append new text and | ||
| 743 | extract objects from the stream accumulated so far (both of these | ||
| 744 | functions are optional). | ||
| 745 | |||
| 746 | If C<$string> is given, then this string is appended to the already | ||
| 747 | existing JSON fragment stored in the C<$json> object. | ||
| 748 | |||
| 749 | After that, if the function is called in void context, it will simply | ||
| 750 | return without doing anything further. This can be used to add more text | ||
| 751 | in as many chunks as you want. | ||
| 752 | |||
| 753 | If the method is called in scalar context, then it will try to extract | ||
| 754 | exactly I<one> JSON object. If that is successful, it will return this | ||
| 755 | object, otherwise it will return C<undef>. If there is a parse error, | ||
| 756 | this method will croak just as C<decode> would do (one can then use | ||
| 757 | C<incr_skip> to skip the erroneous part). This is the most common way of | ||
| 758 | using the method. | ||
| 759 | |||
| 760 | And finally, in list context, it will try to extract as many objects | ||
| 761 | from the stream as it can find and return them, or the empty list | ||
| 762 | otherwise. For this to work, there must be no separators (other than | ||
| 763 | whitespace) between the JSON objects or arrays, instead they must be | ||
| 764 | concatenated back-to-back. If an error occurs, an exception will be | ||
| 765 | raised as in the scalar context case. Note that in this case, any | ||
| 766 | previously-parsed JSON texts will be lost. | ||
| 767 | |||
| 768 | Example: Parse some JSON arrays/objects in a given string and return | ||
| 769 | them. | ||
| 770 | |||
| 771 | my @objs = JSON::XS->new->incr_parse ("[5][7][1,2]"); | ||
| 772 | |||
| 773 | =item $lvalue_string = $json->incr_text | ||
| 774 | |||
| 775 | This method returns the currently stored JSON fragment as an lvalue, that | ||
| 776 | is, you can manipulate it. This I<only> works when a preceding call to | ||
| 777 | C<incr_parse> in I<scalar context> successfully returned an object. Under | ||
| 778 | all other circumstances you must not call this function (I mean it. | ||
| 779 | although in simple tests it might actually work, it I<will> fail under | ||
| 780 | real world conditions). As a special exception, you can also call this | ||
| 781 | method before having parsed anything. | ||
| 782 | |||
| 783 | That means you can only use this function to look at or manipulate text | ||
| 784 | before or after complete JSON objects, not while the parser is in the | ||
| 785 | middle of parsing a JSON object. | ||
| 786 | |||
| 787 | This function is useful in two cases: a) finding the trailing text after a | ||
| 788 | JSON object or b) parsing multiple JSON objects separated by non-JSON text | ||
| 789 | (such as commas). | ||
| 790 | |||
| 791 | =item $json->incr_skip | ||
| 792 | |||
| 793 | This will reset the state of the incremental parser and will remove | ||
| 794 | the parsed text from the input buffer so far. This is useful after | ||
| 795 | C<incr_parse> died, in which case the input buffer and incremental parser | ||
| 796 | state is left unchanged, to skip the text parsed so far and to reset the | ||
| 797 | parse state. | ||
| 798 | |||
| 799 | The difference to C<incr_reset> is that only text until the parse error | ||
| 800 | occurred is removed. | ||
| 801 | |||
| 802 | =item $json->incr_reset | ||
| 803 | |||
| 804 | This completely resets the incremental parser, that is, after this call, | ||
| 805 | it will be as if the parser had never parsed anything. | ||
| 806 | |||
| 807 | This is useful if you want to repeatedly parse JSON objects and want to | ||
| 808 | ignore any trailing data, which means you have to reset the parser after | ||
| 809 | each successful decode. | ||
| 810 | |||
| 811 | =back | ||
| 812 | |||
| 813 | =head2 LIMITATIONS | ||
| 814 | |||
| 815 | The incremental parser is a non-exact parser: it works by gathering as | ||
| 816 | much text as possible that I<could> be a valid JSON text, followed by | ||
| 817 | trying to decode it. | ||
| 818 | |||
| 819 | That means it sometimes needs to read more data than strictly necessary to | ||
| 820 | diagnose an invalid JSON text. For example, after parsing the following | ||
| 821 | fragment, the parser I<could> stop with an error, as this fragment | ||
| 822 | I<cannot> be the beginning of a valid JSON text: | ||
| 823 | |||
| 824 | [, | ||
| 825 | |||
| 826 | In reality, however, the parser might continue to read data until a | ||
| 827 | length limit is exceeded or it finds a closing bracket. | ||
| 828 | |||
| 829 | =head2 EXAMPLES | ||
| 830 | |||
| 831 | Some examples will make all this clearer. First, a simple example that | ||
| 832 | works similarly to C<decode_prefix>: We want to decode the JSON object at | ||
| 833 | the start of a string and identify the portion after the JSON object: | ||
| 834 | |||
| 835 | my $text = "[1,2,3] hello"; | ||
| 836 | |||
| 837 | my $json = new JSON::XS; | ||
| 838 | |||
| 839 | my $obj = $json->incr_parse ($text) | ||
| 840 | or die "expected JSON object or array at beginning of string"; | ||
| 841 | |||
| 842 | my $tail = $json->incr_text; | ||
| 843 | # $tail now contains " hello" | ||
| 844 | |||
| 845 | Easy, isn't it? | ||
| 846 | |||
| 847 | Now for a more complicated example: Imagine a hypothetical protocol where | ||
| 848 | you read some requests from a TCP stream, and each request is a JSON | ||
| 849 | array, without any separation between them (in fact, it is often useful to | ||
| 850 | use newlines as "separators", as these get interpreted as whitespace at | ||
| 851 | the start of the JSON text, which makes it possible to test said protocol | ||
| 852 | with C<telnet>...). | ||
| 853 | |||
| 854 | Here is how you'd do it (it is trivial to write this in an event-based | ||
| 855 | manner): | ||
| 856 | |||
| 857 | my $json = new JSON::XS; | ||
| 858 | |||
| 859 | # read some data from the socket | ||
| 860 | while (sysread $socket, my $buf, 4096) { | ||
| 861 | |||
| 862 | # split and decode as many requests as possible | ||
| 863 | for my $request ($json->incr_parse ($buf)) { | ||
| 864 | # act on the $request | ||
| 865 | } | ||
| 866 | } | ||
| 867 | |||
| 868 | Another complicated example: Assume you have a string with JSON objects | ||
| 869 | or arrays, all separated by (optional) comma characters (e.g. C<[1],[2], | ||
| 870 | [3]>). To parse them, we have to skip the commas between the JSON texts, | ||
| 871 | and here is where the lvalue-ness of C<incr_text> comes in useful: | ||
| 872 | |||
| 873 | my $text = "[1],[2], [3]"; | ||
| 874 | my $json = new JSON::XS; | ||
| 875 | |||
| 876 | # void context, so no parsing done | ||
| 877 | $json->incr_parse ($text); | ||
| 878 | |||
| 879 | # now extract as many objects as possible. note the | ||
| 880 | # use of scalar context so incr_text can be called. | ||
| 881 | while (my $obj = $json->incr_parse) { | ||
| 882 | # do something with $obj | ||
| 883 | |||
| 884 | # now skip the optional comma | ||
| 885 | $json->incr_text =~ s/^ \s* , //x; | ||
| 886 | } | ||
| 887 | |||
| 888 | Now lets go for a very complex example: Assume that you have a gigantic | ||
| 889 | JSON array-of-objects, many gigabytes in size, and you want to parse it, | ||
| 890 | but you cannot load it into memory fully (this has actually happened in | ||
| 891 | the real world :). | ||
| 892 | |||
| 893 | Well, you lost, you have to implement your own JSON parser. But JSON::XS | ||
| 894 | can still help you: You implement a (very simple) array parser and let | ||
| 895 | JSON decode the array elements, which are all full JSON objects on their | ||
| 896 | own (this wouldn't work if the array elements could be JSON numbers, for | ||
| 897 | example): | ||
| 898 | |||
| 899 | my $json = new JSON::XS; | ||
| 900 | |||
| 901 | # open the monster | ||
| 902 | open my $fh, "<bigfile.json" | ||
| 903 | or die "bigfile: $!"; | ||
| 904 | |||
| 905 | # first parse the initial "[" | ||
| 906 | for (;;) { | ||
| 907 | sysread $fh, my $buf, 65536 | ||
| 908 | or die "read error: $!"; | ||
| 909 | $json->incr_parse ($buf); # void context, so no parsing | ||
| 910 | |||
| 911 | # Exit the loop once we found and removed(!) the initial "[". | ||
| 912 | # In essence, we are (ab-)using the $json object as a simple scalar | ||
| 913 | # we append data to. | ||
| 914 | last if $json->incr_text =~ s/^ \s* \[ //x; | ||
| 915 | } | ||
| 916 | |||
| 917 | # now we have the skipped the initial "[", so continue | ||
| 918 | # parsing all the elements. | ||
| 919 | for (;;) { | ||
| 920 | # in this loop we read data until we got a single JSON object | ||
| 921 | for (;;) { | ||
| 922 | if (my $obj = $json->incr_parse) { | ||
| 923 | # do something with $obj | ||
| 924 | last; | ||
| 925 | } | ||
| 926 | |||
| 927 | # add more data | ||
| 928 | sysread $fh, my $buf, 65536 | ||
| 929 | or die "read error: $!"; | ||
| 930 | $json->incr_parse ($buf); # void context, so no parsing | ||
| 931 | } | ||
| 932 | |||
| 933 | # in this loop we read data until we either found and parsed the | ||
| 934 | # separating "," between elements, or the final "]" | ||
| 935 | for (;;) { | ||
| 936 | # first skip whitespace | ||
| 937 | $json->incr_text =~ s/^\s*//; | ||
| 938 | |||
| 939 | # if we find "]", we are done | ||
| 940 | if ($json->incr_text =~ s/^\]//) { | ||
| 941 | print "finished.\n"; | ||
| 942 | exit; | ||
| 943 | } | ||
| 944 | |||
| 945 | # if we find ",", we can continue with the next element | ||
| 946 | if ($json->incr_text =~ s/^,//) { | ||
| 947 | last; | ||
| 948 | } | ||
| 949 | |||
| 950 | # if we find anything else, we have a parse error! | ||
| 951 | if (length $json->incr_text) { | ||
| 952 | die "parse error near ", $json->incr_text; | ||
| 953 | } | ||
| 954 | |||
| 955 | # else add more data | ||
| 956 | sysread $fh, my $buf, 65536 | ||
| 957 | or die "read error: $!"; | ||
| 958 | $json->incr_parse ($buf); # void context, so no parsing | ||
| 959 | } | ||
| 960 | |||
| 961 | This is a complex example, but most of the complexity comes from the fact | ||
| 962 | that we are trying to be correct (bear with me if I am wrong, I never ran | ||
| 963 | the above example :). | ||
| 964 | |||
| 965 | |||
| 966 | |||
| 967 | =head1 MAPPING | ||
| 968 | |||
| 969 | This section describes how JSON::XS maps Perl values to JSON values and | ||
| 970 | vice versa. These mappings are designed to "do the right thing" in most | ||
| 971 | circumstances automatically, preserving round-tripping characteristics | ||
| 972 | (what you put in comes out as something equivalent). | ||
| 973 | |||
| 974 | For the more enlightened: note that in the following descriptions, | ||
| 975 | lowercase I<perl> refers to the Perl interpreter, while uppercase I<Perl> | ||
| 976 | refers to the abstract Perl language itself. | ||
| 977 | |||
| 978 | |||
| 979 | =head2 JSON -> PERL | ||
| 980 | |||
| 981 | =over | ||
| 982 | |||
| 983 | =item object | ||
| 984 | |||
| 985 | A JSON object becomes a reference to a hash in Perl. No ordering of object | ||
| 986 | keys is preserved (JSON does not preserve object key ordering itself). | ||
| 987 | |||
| 988 | =item array | ||
| 989 | |||
| 990 | A JSON array becomes a reference to an array in Perl. | ||
| 991 | |||
| 992 | =item string | ||
| 993 | |||
| 994 | A JSON string becomes a string scalar in Perl - Unicode codepoints in JSON | ||
| 995 | are represented by the same codepoints in the Perl string, so no manual | ||
| 996 | decoding is necessary. | ||
| 997 | |||
| 998 | =item number | ||
| 999 | |||
| 1000 | A JSON number becomes either an integer, numeric (floating point) or | ||
| 1001 | string scalar in perl, depending on its range and any fractional parts. On | ||
| 1002 | the Perl level, there is no difference between those as Perl handles all | ||
| 1003 | the conversion details, but an integer may take slightly less memory and | ||
| 1004 | might represent more values exactly than floating point numbers. | ||
| 1005 | |||
| 1006 | If the number consists of digits only, JSON::XS will try to represent | ||
| 1007 | it as an integer value. If that fails, it will try to represent it as | ||
| 1008 | a numeric (floating point) value if that is possible without loss of | ||
| 1009 | precision. Otherwise it will preserve the number as a string value (in | ||
| 1010 | which case you lose roundtripping ability, as the JSON number will be | ||
| 1011 | re-encoded to a JSON string). | ||
| 1012 | |||
| 1013 | Numbers containing a fractional or exponential part will always be | ||
| 1014 | represented as numeric (floating point) values, possibly at a loss of | ||
| 1015 | precision (in which case you might lose perfect roundtripping ability, but | ||
| 1016 | the JSON number will still be re-encoded as a JSON number). | ||
| 1017 | |||
| 1018 | Note that precision is not accuracy - binary floating point values cannot | ||
| 1019 | represent most decimal fractions exactly, and when converting from and to | ||
| 1020 | floating point, JSON::XS only guarantees precision up to but not including | ||
| 1021 | the least significant bit. | ||
| 1022 | |||
| 1023 | =item true, false | ||
| 1024 | |||
| 1025 | These JSON atoms become C<Types::Serialiser::true> and | ||
| 1026 | C<Types::Serialiser::false>, respectively. They are overloaded to act | ||
| 1027 | almost exactly like the numbers C<1> and C<0>. You can check whether | ||
| 1028 | a scalar is a JSON boolean by using the C<Types::Serialiser::is_bool> | ||
| 1029 | function (after C<use Types::Serialier>, of course). | ||
| 1030 | |||
| 1031 | =item null | ||
| 1032 | |||
| 1033 | A JSON null atom becomes C<undef> in Perl. | ||
| 1034 | |||
| 1035 | =item shell-style comments (C<< # I<text> >>) | ||
| 1036 | |||
| 1037 | As a nonstandard extension to the JSON syntax that is enabled by the | ||
| 1038 | C<relaxed> setting, shell-style comments are allowed. They can start | ||
| 1039 | anywhere outside strings and go till the end of the line. | ||
| 1040 | |||
| 1041 | =item tagged values (C<< (I<tag>)I<value> >>). | ||
| 1042 | |||
| 1043 | Another nonstandard extension to the JSON syntax, enabled with the | ||
| 1044 | C<allow_tags> setting, are tagged values. In this implementation, the | ||
| 1045 | I<tag> must be a perl package/class name encoded as a JSON string, and the | ||
| 1046 | I<value> must be a JSON array encoding optional constructor arguments. | ||
| 1047 | |||
| 1048 | See L<OBJECT SERIALISATION>, below, for details. | ||
| 1049 | |||
| 1050 | =back | ||
| 1051 | |||
| 1052 | |||
| 1053 | =head2 PERL -> JSON | ||
| 1054 | |||
| 1055 | The mapping from Perl to JSON is slightly more difficult, as Perl is a | ||
| 1056 | truly typeless language, so we can only guess which JSON type is meant by | ||
| 1057 | a Perl value. | ||
| 1058 | |||
| 1059 | =over | ||
| 1060 | |||
| 1061 | =item hash references | ||
| 1062 | |||
| 1063 | Perl hash references become JSON objects. As there is no inherent | ||
| 1064 | ordering in hash keys (or JSON objects), they will usually be encoded | ||
| 1065 | in a pseudo-random order. JSON::XS can optionally sort the hash keys | ||
| 1066 | (determined by the I<canonical> flag), so the same datastructure will | ||
| 1067 | serialise to the same JSON text (given same settings and version of | ||
| 1068 | JSON::XS), but this incurs a runtime overhead and is only rarely useful, | ||
| 1069 | e.g. when you want to compare some JSON text against another for equality. | ||
| 1070 | |||
| 1071 | =item array references | ||
| 1072 | |||
| 1073 | Perl array references become JSON arrays. | ||
| 1074 | |||
| 1075 | =item other references | ||
| 1076 | |||
| 1077 | Other unblessed references are generally not allowed and will cause an | ||
| 1078 | exception to be thrown, except for references to the integers C<0> and | ||
| 1079 | C<1>, which get turned into C<false> and C<true> atoms in JSON. | ||
| 1080 | |||
| 1081 | Since C<JSON::XS> uses the boolean model from L<Types::Serialiser>, you | ||
| 1082 | can also C<use Types::Serialiser> and then use C<Types::Serialiser::false> | ||
| 1083 | and C<Types::Serialiser::true> to improve readability. | ||
| 1084 | |||
| 1085 | use Types::Serialiser; | ||
| 1086 | encode_json [\0, Types::Serialiser::true] # yields [false,true] | ||
| 1087 | |||
| 1088 | =item Types::Serialiser::true, Types::Serialiser::false | ||
| 1089 | |||
| 1090 | These special values from the L<Types::Serialiser> module become JSON true | ||
| 1091 | and JSON false values, respectively. You can also use C<\1> and C<\0> | ||
| 1092 | directly if you want. | ||
| 1093 | |||
| 1094 | =item blessed objects | ||
| 1095 | |||
| 1096 | Blessed objects are not directly representable in JSON, but C<JSON::XS> | ||
| 1097 | allows various ways of handling objects. See L<OBJECT SERIALISATION>, | ||
| 1098 | below, for details. | ||
| 1099 | |||
| 1100 | =item simple scalars | ||
| 1101 | |||
| 1102 | Simple Perl scalars (any scalar that is not a reference) are the most | ||
| 1103 | difficult objects to encode: JSON::XS will encode undefined scalars as | ||
| 1104 | JSON C<null> values, scalars that have last been used in a string context | ||
| 1105 | before encoding as JSON strings, and anything else as number value: | ||
| 1106 | |||
| 1107 | # dump as number | ||
| 1108 | encode_json [2] # yields [2] | ||
| 1109 | encode_json [-3.0e17] # yields [-3e+17] | ||
| 1110 | my $value = 5; encode_json [$value] # yields [5] | ||
| 1111 | |||
| 1112 | # used as string, so dump as string | ||
| 1113 | print $value; | ||
| 1114 | encode_json [$value] # yields ["5"] | ||
| 1115 | |||
| 1116 | # undef becomes null | ||
| 1117 | encode_json [undef] # yields [null] | ||
| 1118 | |||
| 1119 | You can force the type to be a JSON string by stringifying it: | ||
| 1120 | |||
| 1121 | my $x = 3.1; # some variable containing a number | ||
| 1122 | "$x"; # stringified | ||
| 1123 | $x .= ""; # another, more awkward way to stringify | ||
| 1124 | print $x; # perl does it for you, too, quite often | ||
| 1125 | |||
| 1126 | You can force the type to be a JSON number by numifying it: | ||
| 1127 | |||
| 1128 | my $x = "3"; # some variable containing a string | ||
| 1129 | $x += 0; # numify it, ensuring it will be dumped as a number | ||
| 1130 | $x *= 1; # same thing, the choice is yours. | ||
| 1131 | |||
| 1132 | You can not currently force the type in other, less obscure, ways. Tell me | ||
| 1133 | if you need this capability (but don't forget to explain why it's needed | ||
| 1134 | :). | ||
| 1135 | |||
| 1136 | Note that numerical precision has the same meaning as under Perl (so | ||
| 1137 | binary to decimal conversion follows the same rules as in Perl, which | ||
| 1138 | can differ to other languages). Also, your perl interpreter might expose | ||
| 1139 | extensions to the floating point numbers of your platform, such as | ||
| 1140 | infinities or NaN's - these cannot be represented in JSON, and it is an | ||
| 1141 | error to pass those in. | ||
| 1142 | |||
| 1143 | =back | ||
| 1144 | |||
| 1145 | =head2 OBJECT SERIALISATION | ||
| 1146 | |||
| 1147 | As JSON cannot directly represent Perl objects, you have to choose between | ||
| 1148 | a pure JSON representation (without the ability to deserialise the object | ||
| 1149 | automatically again), and a nonstandard extension to the JSON syntax, | ||
| 1150 | tagged values. | ||
| 1151 | |||
| 1152 | =head3 SERIALISATION | ||
| 1153 | |||
| 1154 | What happens when C<JSON::XS> encounters a Perl object depends on the | ||
| 1155 | C<allow_blessed>, C<convert_blessed> and C<allow_tags> settings, which are | ||
| 1156 | used in this order: | ||
| 1157 | |||
| 1158 | =over | ||
| 1159 | |||
| 1160 | =item 1. C<allow_tags> is enabled and the object has a C<FREEZE> method. | ||
| 1161 | |||
| 1162 | In this case, C<JSON::XS> uses the L<Types::Serialiser> object | ||
| 1163 | serialisation protocol to create a tagged JSON value, using a nonstandard | ||
| 1164 | extension to the JSON syntax. | ||
| 1165 | |||
| 1166 | This works by invoking the C<FREEZE> method on the object, with the first | ||
| 1167 | argument being the object to serialise, and the second argument being the | ||
| 1168 | constant string C<JSON> to distinguish it from other serialisers. | ||
| 1169 | |||
| 1170 | The C<FREEZE> method can return any number of values (i.e. zero or | ||
| 1171 | more). These values and the paclkage/classname of the object will then be | ||
| 1172 | encoded as a tagged JSON value in the following format: | ||
| 1173 | |||
| 1174 | ("classname")[FREEZE return values...] | ||
| 1175 | |||
| 1176 | e.g.: | ||
| 1177 | |||
| 1178 | ("URI")["http://www.google.com/"] | ||
| 1179 | ("MyDate")[2013,10,29] | ||
| 1180 | ("ImageData::JPEG")["Z3...VlCg=="] | ||
| 1181 | |||
| 1182 | For example, the hypothetical C<My::Object> C<FREEZE> method might use the | ||
| 1183 | objects C<type> and C<id> members to encode the object: | ||
| 1184 | |||
| 1185 | sub My::Object::FREEZE { | ||
| 1186 | my ($self, $serialiser) = @_; | ||
| 1187 | |||
| 1188 | ($self->{type}, $self->{id}) | ||
| 1189 | } | ||
| 1190 | |||
| 1191 | =item 2. C<convert_blessed> is enabled and the object has a C<TO_JSON> method. | ||
| 1192 | |||
| 1193 | In this case, the C<TO_JSON> method of the object is invoked in scalar | ||
| 1194 | context. It must return a single scalar that can be directly encoded into | ||
| 1195 | JSON. This scalar replaces the object in the JSON text. | ||
| 1196 | |||
| 1197 | For example, the following C<TO_JSON> method will convert all L<URI> | ||
| 1198 | objects to JSON strings when serialised. The fact that these values | ||
| 1199 | originally were L<URI> objects is lost. | ||
| 1200 | |||
| 1201 | sub URI::TO_JSON { | ||
| 1202 | my ($uri) = @_; | ||
| 1203 | $uri->as_string | ||
| 1204 | } | ||
| 1205 | |||
| 1206 | =item 3. C<allow_blessed> is enabled. | ||
| 1207 | |||
| 1208 | The object will be serialised as a JSON null value. | ||
| 1209 | |||
| 1210 | =item 4. none of the above | ||
| 1211 | |||
| 1212 | If none of the settings are enabled or the respective methods are missing, | ||
| 1213 | C<JSON::XS> throws an exception. | ||
| 1214 | |||
| 1215 | =back | ||
| 1216 | |||
| 1217 | =head3 DESERIALISATION | ||
| 1218 | |||
| 1219 | For deserialisation there are only two cases to consider: either | ||
| 1220 | nonstandard tagging was used, in which case C<allow_tags> decides, | ||
| 1221 | or objects cannot be automatically be deserialised, in which | ||
| 1222 | case you can use postprocessing or the C<filter_json_object> or | ||
| 1223 | C<filter_json_single_key_object> callbacks to get some real objects our of | ||
| 1224 | your JSON. | ||
| 1225 | |||
| 1226 | This section only considers the tagged value case: I a tagged JSON object | ||
| 1227 | is encountered during decoding and C<allow_tags> is disabled, a parse | ||
| 1228 | error will result (as if tagged values were not part of the grammar). | ||
| 1229 | |||
| 1230 | If C<allow_tags> is enabled, C<JSON::XS> will look up the C<THAW> method | ||
| 1231 | of the package/classname used during serialisation (it will not attempt | ||
| 1232 | to load the package as a Perl module). If there is no such method, the | ||
| 1233 | decoding will fail with an error. | ||
| 1234 | |||
| 1235 | Otherwise, the C<THAW> method is invoked with the classname as first | ||
| 1236 | argument, the constant string C<JSON> as second argument, and all the | ||
| 1237 | values from the JSON array (the values originally returned by the | ||
| 1238 | C<FREEZE> method) as remaining arguments. | ||
| 1239 | |||
| 1240 | The method must then return the object. While technically you can return | ||
| 1241 | any Perl scalar, you might have to enable the C<enable_nonref> setting to | ||
| 1242 | make that work in all cases, so better return an actual blessed reference. | ||
| 1243 | |||
| 1244 | As an example, let's implement a C<THAW> function that regenerates the | ||
| 1245 | C<My::Object> from the C<FREEZE> example earlier: | ||
| 1246 | |||
| 1247 | sub My::Object::THAW { | ||
| 1248 | my ($class, $serialiser, $type, $id) = @_; | ||
| 1249 | |||
| 1250 | $class->new (type => $type, id => $id) | ||
| 1251 | } | ||
| 1252 | |||
| 1253 | |||
| 1254 | =head1 ENCODING/CODESET FLAG NOTES | ||
| 1255 | |||
| 1256 | The interested reader might have seen a number of flags that signify | ||
| 1257 | encodings or codesets - C<utf8>, C<latin1> and C<ascii>. There seems to be | ||
| 1258 | some confusion on what these do, so here is a short comparison: | ||
| 1259 | |||
| 1260 | C<utf8> controls whether the JSON text created by C<encode> (and expected | ||
| 1261 | by C<decode>) is UTF-8 encoded or not, while C<latin1> and C<ascii> only | ||
| 1262 | control whether C<encode> escapes character values outside their respective | ||
| 1263 | codeset range. Neither of these flags conflict with each other, although | ||
| 1264 | some combinations make less sense than others. | ||
| 1265 | |||
| 1266 | Care has been taken to make all flags symmetrical with respect to | ||
| 1267 | C<encode> and C<decode>, that is, texts encoded with any combination of | ||
| 1268 | these flag values will be correctly decoded when the same flags are used | ||
| 1269 | - in general, if you use different flag settings while encoding vs. when | ||
| 1270 | decoding you likely have a bug somewhere. | ||
| 1271 | |||
| 1272 | Below comes a verbose discussion of these flags. Note that a "codeset" is | ||
| 1273 | simply an abstract set of character-codepoint pairs, while an encoding | ||
| 1274 | takes those codepoint numbers and I<encodes> them, in our case into | ||
| 1275 | octets. Unicode is (among other things) a codeset, UTF-8 is an encoding, | ||
| 1276 | and ISO-8859-1 (= latin 1) and ASCII are both codesets I<and> encodings at | ||
| 1277 | the same time, which can be confusing. | ||
| 1278 | |||
| 1279 | =over | ||
| 1280 | |||
| 1281 | =item C<utf8> flag disabled | ||
| 1282 | |||
| 1283 | When C<utf8> is disabled (the default), then C<encode>/C<decode> generate | ||
| 1284 | and expect Unicode strings, that is, characters with high ordinal Unicode | ||
| 1285 | values (> 255) will be encoded as such characters, and likewise such | ||
| 1286 | characters are decoded as-is, no changes to them will be done, except | ||
| 1287 | "(re-)interpreting" them as Unicode codepoints or Unicode characters, | ||
| 1288 | respectively (to Perl, these are the same thing in strings unless you do | ||
| 1289 | funny/weird/dumb stuff). | ||
| 1290 | |||
| 1291 | This is useful when you want to do the encoding yourself (e.g. when you | ||
| 1292 | want to have UTF-16 encoded JSON texts) or when some other layer does | ||
| 1293 | the encoding for you (for example, when printing to a terminal using a | ||
| 1294 | filehandle that transparently encodes to UTF-8 you certainly do NOT want | ||
| 1295 | to UTF-8 encode your data first and have Perl encode it another time). | ||
| 1296 | |||
| 1297 | =item C<utf8> flag enabled | ||
| 1298 | |||
| 1299 | If the C<utf8>-flag is enabled, C<encode>/C<decode> will encode all | ||
| 1300 | characters using the corresponding UTF-8 multi-byte sequence, and will | ||
| 1301 | expect your input strings to be encoded as UTF-8, that is, no "character" | ||
| 1302 | of the input string must have any value > 255, as UTF-8 does not allow | ||
| 1303 | that. | ||
| 1304 | |||
| 1305 | The C<utf8> flag therefore switches between two modes: disabled means you | ||
| 1306 | will get a Unicode string in Perl, enabled means you get a UTF-8 encoded | ||
| 1307 | octet/binary string in Perl. | ||
| 1308 | |||
| 1309 | =item C<latin1> or C<ascii> flags enabled | ||
| 1310 | |||
| 1311 | With C<latin1> (or C<ascii>) enabled, C<encode> will escape characters | ||
| 1312 | with ordinal values > 255 (> 127 with C<ascii>) and encode the remaining | ||
| 1313 | characters as specified by the C<utf8> flag. | ||
| 1314 | |||
| 1315 | If C<utf8> is disabled, then the result is also correctly encoded in those | ||
| 1316 | character sets (as both are proper subsets of Unicode, meaning that a | ||
| 1317 | Unicode string with all character values < 256 is the same thing as a | ||
| 1318 | ISO-8859-1 string, and a Unicode string with all character values < 128 is | ||
| 1319 | the same thing as an ASCII string in Perl). | ||
| 1320 | |||
| 1321 | If C<utf8> is enabled, you still get a correct UTF-8-encoded string, | ||
| 1322 | regardless of these flags, just some more characters will be escaped using | ||
| 1323 | C<\uXXXX> then before. | ||
| 1324 | |||
| 1325 | Note that ISO-8859-1-I<encoded> strings are not compatible with UTF-8 | ||
| 1326 | encoding, while ASCII-encoded strings are. That is because the ISO-8859-1 | ||
| 1327 | encoding is NOT a subset of UTF-8 (despite the ISO-8859-1 I<codeset> being | ||
| 1328 | a subset of Unicode), while ASCII is. | ||
| 1329 | |||
| 1330 | Surprisingly, C<decode> will ignore these flags and so treat all input | ||
| 1331 | values as governed by the C<utf8> flag. If it is disabled, this allows you | ||
| 1332 | to decode ISO-8859-1- and ASCII-encoded strings, as both strict subsets of | ||
| 1333 | Unicode. If it is enabled, you can correctly decode UTF-8 encoded strings. | ||
| 1334 | |||
| 1335 | So neither C<latin1> nor C<ascii> are incompatible with the C<utf8> flag - | ||
| 1336 | they only govern when the JSON output engine escapes a character or not. | ||
| 1337 | |||
| 1338 | The main use for C<latin1> is to relatively efficiently store binary data | ||
| 1339 | as JSON, at the expense of breaking compatibility with most JSON decoders. | ||
| 1340 | |||
| 1341 | The main use for C<ascii> is to force the output to not contain characters | ||
| 1342 | with values > 127, which means you can interpret the resulting string | ||
| 1343 | as UTF-8, ISO-8859-1, ASCII, KOI8-R or most about any character set and | ||
| 1344 | 8-bit-encoding, and still get the same data structure back. This is useful | ||
| 1345 | when your channel for JSON transfer is not 8-bit clean or the encoding | ||
| 1346 | might be mangled in between (e.g. in mail), and works because ASCII is a | ||
| 1347 | proper subset of most 8-bit and multibyte encodings in use in the world. | ||
| 1348 | |||
| 1349 | =back | ||
| 1350 | |||
| 1351 | |||
| 1352 | =head2 JSON and ECMAscript | ||
| 1353 | |||
| 1354 | JSON syntax is based on how literals are represented in javascript (the | ||
| 1355 | not-standardised predecessor of ECMAscript) which is presumably why it is | ||
| 1356 | called "JavaScript Object Notation". | ||
| 1357 | |||
| 1358 | However, JSON is not a subset (and also not a superset of course) of | ||
| 1359 | ECMAscript (the standard) or javascript (whatever browsers actually | ||
| 1360 | implement). | ||
| 1361 | |||
| 1362 | If you want to use javascript's C<eval> function to "parse" JSON, you | ||
| 1363 | might run into parse errors for valid JSON texts, or the resulting data | ||
| 1364 | structure might not be queryable: | ||
| 1365 | |||
| 1366 | One of the problems is that U+2028 and U+2029 are valid characters inside | ||
| 1367 | JSON strings, but are not allowed in ECMAscript string literals, so the | ||
| 1368 | following Perl fragment will not output something that can be guaranteed | ||
| 1369 | to be parsable by javascript's C<eval>: | ||
| 1370 | |||
| 1371 | use JSON::XS; | ||
| 1372 | |||
| 1373 | print encode_json [chr 0x2028]; | ||
| 1374 | |||
| 1375 | The right fix for this is to use a proper JSON parser in your javascript | ||
| 1376 | programs, and not rely on C<eval> (see for example Douglas Crockford's | ||
| 1377 | F<json2.js> parser). | ||
| 1378 | |||
| 1379 | If this is not an option, you can, as a stop-gap measure, simply encode to | ||
| 1380 | ASCII-only JSON: | ||
| 1381 | |||
| 1382 | use JSON::XS; | ||
| 1383 | |||
| 1384 | print JSON::XS->new->ascii->encode ([chr 0x2028]); | ||
| 1385 | |||
| 1386 | Note that this will enlarge the resulting JSON text quite a bit if you | ||
| 1387 | have many non-ASCII characters. You might be tempted to run some regexes | ||
| 1388 | to only escape U+2028 and U+2029, e.g.: | ||
| 1389 | |||
| 1390 | # DO NOT USE THIS! | ||
| 1391 | my $json = JSON::XS->new->utf8->encode ([chr 0x2028]); | ||
| 1392 | $json =~ s/\xe2\x80\xa8/\\u2028/g; # escape U+2028 | ||
| 1393 | $json =~ s/\xe2\x80\xa9/\\u2029/g; # escape U+2029 | ||
| 1394 | print $json; | ||
| 1395 | |||
| 1396 | Note that I<this is a bad idea>: the above only works for U+2028 and | ||
| 1397 | U+2029 and thus only for fully ECMAscript-compliant parsers. Many existing | ||
| 1398 | javascript implementations, however, have issues with other characters as | ||
| 1399 | well - using C<eval> naively simply I<will> cause problems. | ||
| 1400 | |||
| 1401 | Another problem is that some javascript implementations reserve | ||
| 1402 | some property names for their own purposes (which probably makes | ||
| 1403 | them non-ECMAscript-compliant). For example, Iceweasel reserves the | ||
| 1404 | C<__proto__> property name for its own purposes. | ||
| 1405 | |||
| 1406 | If that is a problem, you could parse try to filter the resulting JSON | ||
| 1407 | output for these property strings, e.g.: | ||
| 1408 | |||
| 1409 | $json =~ s/"__proto__"\s*:/"__proto__renamed":/g; | ||
| 1410 | |||
| 1411 | This works because C<__proto__> is not valid outside of strings, so every | ||
| 1412 | occurrence of C<"__proto__"\s*:> must be a string used as property name. | ||
| 1413 | |||
| 1414 | If you know of other incompatibilities, please let me know. | ||
| 1415 | |||
| 1416 | |||
| 1417 | =head2 JSON and YAML | ||
| 1418 | |||
| 1419 | You often hear that JSON is a subset of YAML. This is, however, a mass | ||
| 1420 | hysteria(*) and very far from the truth (as of the time of this writing), | ||
| 1421 | so let me state it clearly: I<in general, there is no way to configure | ||
| 1422 | JSON::XS to output a data structure as valid YAML> that works in all | ||
| 1423 | cases. | ||
| 1424 | |||
| 1425 | If you really must use JSON::XS to generate YAML, you should use this | ||
| 1426 | algorithm (subject to change in future versions): | ||
| 1427 | |||
| 1428 | my $to_yaml = JSON::XS->new->utf8->space_after (1); | ||
| 1429 | my $yaml = $to_yaml->encode ($ref) . "\n"; | ||
| 1430 | |||
| 1431 | This will I<usually> generate JSON texts that also parse as valid | ||
| 1432 | YAML. Please note that YAML has hardcoded limits on (simple) object key | ||
| 1433 | lengths that JSON doesn't have and also has different and incompatible | ||
| 1434 | unicode character escape syntax, so you should make sure that your hash | ||
| 1435 | keys are noticeably shorter than the 1024 "stream characters" YAML allows | ||
| 1436 | and that you do not have characters with codepoint values outside the | ||
| 1437 | Unicode BMP (basic multilingual page). YAML also does not allow C<\/> | ||
| 1438 | sequences in strings (which JSON::XS does not I<currently> generate, but | ||
| 1439 | other JSON generators might). | ||
| 1440 | |||
| 1441 | There might be other incompatibilities that I am not aware of (or the YAML | ||
| 1442 | specification has been changed yet again - it does so quite often). In | ||
| 1443 | general you should not try to generate YAML with a JSON generator or vice | ||
| 1444 | versa, or try to parse JSON with a YAML parser or vice versa: chances are | ||
| 1445 | high that you will run into severe interoperability problems when you | ||
| 1446 | least expect it. | ||
| 1447 | |||
| 1448 | =over | ||
| 1449 | |||
| 1450 | =item (*) | ||
| 1451 | |||
| 1452 | I have been pressured multiple times by Brian Ingerson (one of the | ||
| 1453 | authors of the YAML specification) to remove this paragraph, despite him | ||
| 1454 | acknowledging that the actual incompatibilities exist. As I was personally | ||
| 1455 | bitten by this "JSON is YAML" lie, I refused and said I will continue to | ||
| 1456 | educate people about these issues, so others do not run into the same | ||
| 1457 | problem again and again. After this, Brian called me a (quote)I<complete | ||
| 1458 | and worthless idiot>(unquote). | ||
| 1459 | |||
| 1460 | In my opinion, instead of pressuring and insulting people who actually | ||
| 1461 | clarify issues with YAML and the wrong statements of some of its | ||
| 1462 | proponents, I would kindly suggest reading the JSON spec (which is not | ||
| 1463 | that difficult or long) and finally make YAML compatible to it, and | ||
| 1464 | educating users about the changes, instead of spreading lies about the | ||
| 1465 | real compatibility for many I<years> and trying to silence people who | ||
| 1466 | point out that it isn't true. | ||
| 1467 | |||
| 1468 | Addendum/2009: the YAML 1.2 spec is still incompatible with JSON, even | ||
| 1469 | though the incompatibilities have been documented (and are known to Brian) | ||
| 1470 | for many years and the spec makes explicit claims that YAML is a superset | ||
| 1471 | of JSON. It would be so easy to fix, but apparently, bullying people and | ||
| 1472 | corrupting userdata is so much easier. | ||
| 1473 | |||
| 1474 | =back | ||
| 1475 | |||
| 1476 | |||
| 1477 | =head2 SPEED | ||
| 1478 | |||
| 1479 | It seems that JSON::XS is surprisingly fast, as shown in the following | ||
| 1480 | tables. They have been generated with the help of the C<eg/bench> program | ||
| 1481 | in the JSON::XS distribution, to make it easy to compare on your own | ||
| 1482 | system. | ||
| 1483 | |||
| 1484 | First comes a comparison between various modules using | ||
| 1485 | a very short single-line JSON string (also available at | ||
| 1486 | L<http://dist.schmorp.de/misc/json/short.json>). | ||
| 1487 | |||
| 1488 | {"method": "handleMessage", "params": ["user1", | ||
| 1489 | "we were just talking"], "id": null, "array":[1,11,234,-5,1e5,1e7, | ||
| 1490 | 1, 0]} | ||
| 1491 | |||
| 1492 | It shows the number of encodes/decodes per second (JSON::XS uses | ||
| 1493 | the functional interface, while JSON::XS/2 uses the OO interface | ||
| 1494 | with pretty-printing and hashkey sorting enabled, JSON::XS/3 enables | ||
| 1495 | shrink. JSON::DWIW/DS uses the deserialise function, while JSON::DWIW::FJ | ||
| 1496 | uses the from_json method). Higher is better: | ||
| 1497 | |||
| 1498 | module | encode | decode | | ||
| 1499 | --------------|------------|------------| | ||
| 1500 | JSON::DWIW/DS | 86302.551 | 102300.098 | | ||
| 1501 | JSON::DWIW/FJ | 86302.551 | 75983.768 | | ||
| 1502 | JSON::PP | 15827.562 | 6638.658 | | ||
| 1503 | JSON::Syck | 63358.066 | 47662.545 | | ||
| 1504 | JSON::XS | 511500.488 | 511500.488 | | ||
| 1505 | JSON::XS/2 | 291271.111 | 388361.481 | | ||
| 1506 | JSON::XS/3 | 361577.931 | 361577.931 | | ||
| 1507 | Storable | 66788.280 | 265462.278 | | ||
| 1508 | --------------+------------+------------+ | ||
| 1509 | |||
| 1510 | That is, JSON::XS is almost six times faster than JSON::DWIW on encoding, | ||
| 1511 | about five times faster on decoding, and over thirty to seventy times | ||
| 1512 | faster than JSON's pure perl implementation. It also compares favourably | ||
| 1513 | to Storable for small amounts of data. | ||
| 1514 | |||
| 1515 | Using a longer test string (roughly 18KB, generated from Yahoo! Locals | ||
| 1516 | search API (L<http://dist.schmorp.de/misc/json/long.json>). | ||
| 1517 | |||
| 1518 | module | encode | decode | | ||
| 1519 | --------------|------------|------------| | ||
| 1520 | JSON::DWIW/DS | 1647.927 | 2673.916 | | ||
| 1521 | JSON::DWIW/FJ | 1630.249 | 2596.128 | | ||
| 1522 | JSON::PP | 400.640 | 62.311 | | ||
| 1523 | JSON::Syck | 1481.040 | 1524.869 | | ||
| 1524 | JSON::XS | 20661.596 | 9541.183 | | ||
| 1525 | JSON::XS/2 | 10683.403 | 9416.938 | | ||
| 1526 | JSON::XS/3 | 20661.596 | 9400.054 | | ||
| 1527 | Storable | 19765.806 | 10000.725 | | ||
| 1528 | --------------+------------+------------+ | ||
| 1529 | |||
| 1530 | Again, JSON::XS leads by far (except for Storable which non-surprisingly | ||
| 1531 | decodes a bit faster). | ||
| 1532 | |||
| 1533 | On large strings containing lots of high Unicode characters, some modules | ||
| 1534 | (such as JSON::PC) seem to decode faster than JSON::XS, but the result | ||
| 1535 | will be broken due to missing (or wrong) Unicode handling. Others refuse | ||
| 1536 | to decode or encode properly, so it was impossible to prepare a fair | ||
| 1537 | comparison table for that case. | ||
| 1538 | |||
| 1539 | |||
| 1540 | =head1 SECURITY CONSIDERATIONS | ||
| 1541 | |||
| 1542 | When you are using JSON in a protocol, talking to untrusted potentially | ||
| 1543 | hostile creatures requires relatively few measures. | ||
| 1544 | |||
| 1545 | First of all, your JSON decoder should be secure, that is, should not have | ||
| 1546 | any buffer overflows. Obviously, this module should ensure that and I am | ||
| 1547 | trying hard on making that true, but you never know. | ||
| 1548 | |||
| 1549 | Second, you need to avoid resource-starving attacks. That means you should | ||
| 1550 | limit the size of JSON texts you accept, or make sure then when your | ||
| 1551 | resources run out, that's just fine (e.g. by using a separate process that | ||
| 1552 | can crash safely). The size of a JSON text in octets or characters is | ||
| 1553 | usually a good indication of the size of the resources required to decode | ||
| 1554 | it into a Perl structure. While JSON::XS can check the size of the JSON | ||
| 1555 | text, it might be too late when you already have it in memory, so you | ||
| 1556 | might want to check the size before you accept the string. | ||
| 1557 | |||
| 1558 | Third, JSON::XS recurses using the C stack when decoding objects and | ||
| 1559 | arrays. The C stack is a limited resource: for instance, on my amd64 | ||
| 1560 | machine with 8MB of stack size I can decode around 180k nested arrays but | ||
| 1561 | only 14k nested JSON objects (due to perl itself recursing deeply on croak | ||
| 1562 | to free the temporary). If that is exceeded, the program crashes. To be | ||
| 1563 | conservative, the default nesting limit is set to 512. If your process | ||
| 1564 | has a smaller stack, you should adjust this setting accordingly with the | ||
| 1565 | C<max_depth> method. | ||
| 1566 | |||
| 1567 | Something else could bomb you, too, that I forgot to think of. In that | ||
| 1568 | case, you get to keep the pieces. I am always open for hints, though... | ||
| 1569 | |||
| 1570 | Also keep in mind that JSON::XS might leak contents of your Perl data | ||
| 1571 | structures in its error messages, so when you serialise sensitive | ||
| 1572 | information you might want to make sure that exceptions thrown by JSON::XS | ||
| 1573 | will not end up in front of untrusted eyes. | ||
| 1574 | |||
| 1575 | If you are using JSON::XS to return packets to consumption | ||
| 1576 | by JavaScript scripts in a browser you should have a look at | ||
| 1577 | L<http://blog.archive.jpsykes.com/47/practical-csrf-and-json-security/> to | ||
| 1578 | see whether you are vulnerable to some common attack vectors (which really | ||
| 1579 | are browser design bugs, but it is still you who will have to deal with | ||
| 1580 | it, as major browser developers care only for features, not about getting | ||
| 1581 | security right). | ||
| 1582 | |||
| 1583 | |||
| 1584 | =head2 "OLD" VS. "NEW" JSON (RFC4627 VS. RFC7159) | ||
| 1585 | |||
| 1586 | JSON originally required JSON texts to represent an array or object - | ||
| 1587 | scalar values were explicitly not allowed. This has changed, and versions | ||
| 1588 | of JSON::XS beginning with C<4.0> reflect this by allowing scalar values | ||
| 1589 | by default. | ||
| 1590 | |||
| 1591 | One reason why one might not want this is that this removes a fundamental | ||
| 1592 | property of JSON texts, namely that they are self-delimited and | ||
| 1593 | self-contained, or in other words, you could take any number of "old" | ||
| 1594 | JSON texts and paste them together, and the result would be unambiguously | ||
| 1595 | parseable: | ||
| 1596 | |||
| 1597 | [1,3]{"k":5}[][null] # four JSON texts, without doubt | ||
| 1598 | |||
| 1599 | By allowing scalars, this property is lost: in the following example, is | ||
| 1600 | this one JSON text (the number 12) or two JSON texts (the numbers 1 and | ||
| 1601 | 2): | ||
| 1602 | |||
| 1603 | 12 # could be 12, or 1 and 2 | ||
| 1604 | |||
| 1605 | Another lost property of "old" JSON is that no lookahead is required to | ||
| 1606 | know the end of a JSON text, i.e. the JSON text definitely ended at the | ||
| 1607 | last C<]> or C<}> character, there was no need to read extra characters. | ||
| 1608 | |||
| 1609 | For example, a viable network protocol with "old" JSON was to simply | ||
| 1610 | exchange JSON texts without delimiter. For "new" JSON, you have to use a | ||
| 1611 | suitable delimiter (such as a newline) after every JSON text or ensure you | ||
| 1612 | never encode/decode scalar values. | ||
| 1613 | |||
| 1614 | Most protocols do work by only transferring arrays or objects, and the | ||
| 1615 | easiest way to avoid problems with the "new" JSON definition is to | ||
| 1616 | explicitly disallow scalar values in your encoder and decoder: | ||
| 1617 | |||
| 1618 | $json_coder = JSON::XS->new->allow_nonref (0) | ||
| 1619 | |||
| 1620 | This is a somewhat unhappy situation, and the blame can fully be put on | ||
| 1621 | JSON's inventor, Douglas Crockford, who unilaterally changed the format | ||
| 1622 | in 2006 without consulting the IETF, forcing the IETF to either fork the | ||
| 1623 | format or go with it (as I was told, the IETF wasn't amused). | ||
| 1624 | |||
| 1625 | |||
| 1626 | =head1 RELATIONSHIP WITH I-JSON | ||
| 1627 | |||
| 1628 | JSON is a somewhat sloppily-defined format - it carries around obvious | ||
| 1629 | Javascript baggage, such as not really defining number range, probably | ||
| 1630 | because Javascript only has one type of numbers: IEEE 64 bit floats | ||
| 1631 | ("binary64"). | ||
| 1632 | |||
| 1633 | For this reaosn, RFC7493 defines "Internet JSON", which is a restricted | ||
| 1634 | subset of JSON that is supposedly more interoperable on the internet. | ||
| 1635 | |||
| 1636 | While C<JSON::XS> does not offer specific support for I-JSON, it of course | ||
| 1637 | accepts valid I-JSON and by default implements some of the limitations | ||
| 1638 | of I-JSON, such as parsing numbers as perl numbers, which are usually a | ||
| 1639 | superset of binary64 numbers. | ||
| 1640 | |||
| 1641 | To generate I-JSON, follow these rules: | ||
| 1642 | |||
| 1643 | =over | ||
| 1644 | |||
| 1645 | =item * always generate UTF-8 | ||
| 1646 | |||
| 1647 | I-JSON must be encoded in UTF-8, the default for C<encode_json>. | ||
| 1648 | |||
| 1649 | =item * numbers should be within IEEE 754 binary64 range | ||
| 1650 | |||
| 1651 | Basically all existing perl installations use binary64 to represent | ||
| 1652 | floating point numbers, so all you need to do is to avoid large integers. | ||
| 1653 | |||
| 1654 | =item * objects must not have duplicate keys | ||
| 1655 | |||
| 1656 | This is trivially done, as C<JSON::XS> does not allow duplicate keys. | ||
| 1657 | |||
| 1658 | =item * do not generate scalar JSON texts, use C<< ->allow_nonref (0) >> | ||
| 1659 | |||
| 1660 | I-JSON strongly requests you to only encode arrays and objects into JSON. | ||
| 1661 | |||
| 1662 | =item * times should be strings in ISO 8601 format | ||
| 1663 | |||
| 1664 | There are a myriad of modules on CPAN dealing with ISO 8601 - search for | ||
| 1665 | C<ISO8601> on CPAN and use one. | ||
| 1666 | |||
| 1667 | =item * encode binary data as base64 | ||
| 1668 | |||
| 1669 | While it's tempting to just dump binary data as a string (and let | ||
| 1670 | C<JSON::XS> do the escaping), for I-JSON, it's I<recommended> to encode | ||
| 1671 | binary data as base64. | ||
| 1672 | |||
| 1673 | =back | ||
| 1674 | |||
| 1675 | There are some other considerations - read RFC7493 for the details if | ||
| 1676 | interested. | ||
| 1677 | |||
| 1678 | |||
| 1679 | =head1 INTEROPERABILITY WITH OTHER MODULES | ||
| 1680 | |||
| 1681 | C<JSON::XS> uses the L<Types::Serialiser> module to provide boolean | ||
| 1682 | constants. That means that the JSON true and false values will be | ||
| 1683 | compatible to true and false values of other modules that do the same, | ||
| 1684 | such as L<JSON::PP> and L<CBOR::XS>. | ||
| 1685 | |||
| 1686 | |||
| 1687 | =head1 INTEROPERABILITY WITH OTHER JSON DECODERS | ||
| 1688 | |||
| 1689 | As long as you only serialise data that can be directly expressed in JSON, | ||
| 1690 | C<JSON::XS> is incapable of generating invalid JSON output (modulo bugs, | ||
| 1691 | but C<JSON::XS> has found more bugs in the official JSON testsuite (1) | ||
| 1692 | than the official JSON testsuite has found in C<JSON::XS> (0)). | ||
| 1693 | |||
| 1694 | When you have trouble decoding JSON generated by this module using other | ||
| 1695 | decoders, then it is very likely that you have an encoding mismatch or the | ||
| 1696 | other decoder is broken. | ||
| 1697 | |||
| 1698 | When decoding, C<JSON::XS> is strict by default and will likely catch all | ||
| 1699 | errors. There are currently two settings that change this: C<relaxed> | ||
| 1700 | makes C<JSON::XS> accept (but not generate) some non-standard extensions, | ||
| 1701 | and C<allow_tags> will allow you to encode and decode Perl objects, at the | ||
| 1702 | cost of not outputting valid JSON anymore. | ||
| 1703 | |||
| 1704 | =head2 TAGGED VALUE SYNTAX AND STANDARD JSON EN/DECODERS | ||
| 1705 | |||
| 1706 | When you use C<allow_tags> to use the extended (and also nonstandard and | ||
| 1707 | invalid) JSON syntax for serialised objects, and you still want to decode | ||
| 1708 | the generated text with a standard JSON decoder, you can run a regex to | ||
| 1709 | replace the tagged syntax by standard JSON arrays (it only works for | ||
| 1710 | "normal" package names without comma, newlines or single colons). First, | ||
| 1711 | the readable Perl version: | ||
| 1712 | |||
| 1713 | # if your FREEZE methods return no values, you need this replace first: | ||
| 1714 | $json =~ s/\( \s* (" (?: [^\\":,]+|\\.|::)* ") \s* \) \s* \[\s*\]/[$1]/gx; | ||
| 1715 | |||
| 1716 | # this works for non-empty constructor arg lists: | ||
| 1717 | $json =~ s/\( \s* (" (?: [^\\":,]+|\\.|::)* ") \s* \) \s* \[/[$1,/gx; | ||
| 1718 | |||
| 1719 | And here is a less readable version that is easy to adapt to other | ||
| 1720 | languages: | ||
| 1721 | |||
| 1722 | $json =~ s/\(\s*("([^\\":,]+|\\.|::)*")\s*\)\s*\[/[$1,/g; | ||
| 1723 | |||
| 1724 | Here is an ECMAScript version (same regex): | ||
| 1725 | |||
| 1726 | json = json.replace (/\(\s*("([^\\":,]+|\\.|::)*")\s*\)\s*\[/g, "[$1,"); | ||
| 1727 | |||
| 1728 | Since this syntax converts to standard JSON arrays, it might be hard to | ||
| 1729 | distinguish serialised objects from normal arrays. You can prepend a | ||
| 1730 | "magic number" as first array element to reduce chances of a collision: | ||
| 1731 | |||
| 1732 | $json =~ s/\(\s*("([^\\":,]+|\\.|::)*")\s*\)\s*\[/["XU1peReLzT4ggEllLanBYq4G9VzliwKF",$1,/g; | ||
| 1733 | |||
| 1734 | And after decoding the JSON text, you could walk the data | ||
| 1735 | structure looking for arrays with a first element of | ||
| 1736 | C<XU1peReLzT4ggEllLanBYq4G9VzliwKF>. | ||
| 1737 | |||
| 1738 | The same approach can be used to create the tagged format with another | ||
| 1739 | encoder. First, you create an array with the magic string as first member, | ||
| 1740 | the classname as second, and constructor arguments last, encode it as part | ||
| 1741 | of your JSON structure, and then: | ||
| 1742 | |||
| 1743 | $json =~ s/\[\s*"XU1peReLzT4ggEllLanBYq4G9VzliwKF"\s*,\s*("([^\\":,]+|\\.|::)*")\s*,/($1)[/g; | ||
| 1744 | |||
| 1745 | Again, this has some limitations - the magic string must not be encoded | ||
| 1746 | with character escapes, and the constructor arguments must be non-empty. | ||
| 1747 | |||
| 1748 | |||
| 1749 | =head1 (I-)THREADS | ||
| 1750 | |||
| 1751 | This module is I<not> guaranteed to be ithread (or MULTIPLICITY-) safe | ||
| 1752 | and there are no plans to change this. Note that perl's builtin so-called | ||
| 1753 | threads/ithreads are officially deprecated and should not be used. | ||
| 1754 | |||
| 1755 | |||
| 1756 | =head1 THE PERILS OF SETLOCALE | ||
| 1757 | |||
| 1758 | Sometimes people avoid the Perl locale support and directly call the | ||
| 1759 | system's setlocale function with C<LC_ALL>. | ||
| 1760 | |||
| 1761 | This breaks both perl and modules such as JSON::XS, as stringification of | ||
| 1762 | numbers no longer works correctly (e.g. C<$x = 0.1; print "$x"+1> might | ||
| 1763 | print C<1>, and JSON::XS might output illegal JSON as JSON::XS relies on | ||
| 1764 | perl to stringify numbers). | ||
| 1765 | |||
| 1766 | The solution is simple: don't call C<setlocale>, or use it for only those | ||
| 1767 | categories you need, such as C<LC_MESSAGES> or C<LC_CTYPE>. | ||
| 1768 | |||
| 1769 | If you need C<LC_NUMERIC>, you should enable it only around the code that | ||
| 1770 | actually needs it (avoiding stringification of numbers), and restore it | ||
| 1771 | afterwards. | ||
| 1772 | |||
| 1773 | |||
| 1774 | =head1 SOME HISTORY | ||
| 1775 | |||
| 1776 | At the time this module was created there already were a number of JSON | ||
| 1777 | modules available on CPAN, so what was the reason to write yet another | ||
| 1778 | JSON module? While it seems there are many JSON modules, none of them | ||
| 1779 | correctly handled all corner cases, and in most cases their maintainers | ||
| 1780 | are unresponsive, gone missing, or not listening to bug reports for other | ||
| 1781 | reasons. | ||
| 1782 | |||
| 1783 | Beginning with version 2.0 of the JSON module, when both JSON and | ||
| 1784 | JSON::XS are installed, then JSON will fall back on JSON::XS (this can be | ||
| 1785 | overridden) with no overhead due to emulation (by inheriting constructor | ||
| 1786 | and methods). If JSON::XS is not available, it will fall back to the | ||
| 1787 | compatible JSON::PP module as backend, so using JSON instead of JSON::XS | ||
| 1788 | gives you a portable JSON API that can be fast when you need it and | ||
| 1789 | doesn't require a C compiler when that is a problem. | ||
| 1790 | |||
| 1791 | Somewhere around version 3, this module was forked into | ||
| 1792 | C<Cpanel::JSON::XS>, because its maintainer had serious trouble | ||
| 1793 | understanding JSON and insisted on a fork with many bugs "fixed" that | ||
| 1794 | weren't actually bugs, while spreading FUD about this module without | ||
| 1795 | actually giving any details on his accusations. You be the judge, but | ||
| 1796 | in my personal opinion, if you want quality, you will stay away from | ||
| 1797 | dangerous forks like that. | ||
| 1798 | |||
| 1799 | |||
| 1800 | =head1 BUGS | ||
| 1801 | |||
| 1802 | While the goal of this module is to be correct, that unfortunately does | ||
| 1803 | not mean it's bug-free, only that I think its design is bug-free. If you | ||
| 1804 | keep reporting bugs they will be fixed swiftly, though. | ||
| 1805 | |||
| 1806 | Please refrain from using rt.cpan.org or any other bug reporting | ||
| 1807 | service. I put the contact address into my modules for a reason. | ||
| 1808 | |||
| 1809 | =cut | ||
| 1810 | |||
| 1811 | 25 | ▶ | BEGIN { |
|
Subroutine
BEGIN: called CC=1 SCAR=0.0
| |||
| 1812 | 25 | *true = \$Types::Serialiser::true; | |
| 1813 | 25 | *true = \&Types::Serialiser::true; | |
| 1814 | 25 | *false = \$Types::Serialiser::false; | |
| 1815 | 25 | *false = \&Types::Serialiser::false; | |
| 1816 | 25 | *is_bool = \&Types::Serialiser::is_bool; | |
| 1817 | |||
| 1818 | 25 | *JSON::XS::Boolean:: = *Types::Serialiser::Boolean::; | |
| 1819 | } | ||
| 1820 | |||
| 1821 | 25 | XSLoader::load "JSON::XS", $VERSION; | |
| 1822 | |||
| 1823 | =head1 SEE ALSO | ||
| 1824 | |||
| 1825 | The F<json_xs> command line utility for quick experiments. | ||
| 1826 | |||
| 1827 | =head1 AUTHOR | ||
| 1828 | |||
| 1829 | Marc Lehmann <schmorp@schmorp.de> | ||
| 1830 | http://home.schmorp.de/ | ||
| 1831 | |||
| 1832 | =cut | ||
| 1833 | |||
| 1834 | 25 | 1 | |
| 1835 | |||