Merge pull request #388 from pjungkamp/fmts/raw0

Introduce `raw0` format for zero-terminated strings

Author: 01mf02

docs/cli.dj

@@ -40,7 +40,7 @@ For example,
 `jaq --from yaml . myfile.yml`
 parses `myfile.yml` as YAML.
 Possible values of _FORMAT_ include:
-`raw`, `json`, `yaml`, `cbor`, `toml`, `xml`.
+`raw`, `raw0`, `json`, `yaml`, `cbor`, `toml`, `xml`.
 
 jaq automatically chooses the corresponding input format for
 files with the extensions
@@ -100,6 +100,41 @@ See [`--rawfile`](#--rawfile).
 
 This is equivalent to `--from raw`.
 
+{#--raw-input0}
+### `--raw-input0`
+
+Read values like [`--raw-input`](#--raw-input), splitting by NUL (`\0`) instead of newlines.
+For example:
+
+```
+$ printf "Hello\nWorld\0foo" | jaq --raw-input0
+"Hello\nWorld"
+"foo"
+```
+
+When combined with [`--slurp`](#--slurp),
+this collects all inputs into an array.
+For example:
+
+```
+$ printf "Hello\nWorld\0foo" | jaq -sc --raw-input0
+["Hello\nWorld","foo"]
+```
+
+This option can be used to round-trip output from [`--raw-output0`](#--raw-output0):
+
+```
+$ printf '"some string\\nwith a newline" "another string"' | jaq --raw-output0 | jaq --raw-input0
+"some string\nwith a newline"
+"another string"
+```
+
+This is equivalent to `--from raw0`.
+
+::: Compatibility
+`jq` does not have this option.
+:::
+
 {#--slurp}
 ### `-s`, `--slurp`
 
@@ -194,6 +229,26 @@ $ printf '["Hello\\nWorld"]' | jaq -rc
 
 This is equivalent to `--to raw`.
 
+{#--raw-output0}
+### `--raw-output0`
+
+Output values like [`--raw-output`](#--raw-output), terminating them with NUL (`\0`) instead of LF (`\n`).
+
+```
+$ printf '"some string\\nwith a newline" true' | jaq --raw-output0 | jaq -s --raw-input tobytes
+b"some string\nwith a newline\x00true\x00"
+```
+
+Any output value that is a string containing NUL produces an error.
+Note that this does not apply to strings contained in an array or an object.
+
+```
+$ printf '"string with NUL \\u0000"' | jaq --raw-output0 2>/dev/null; echo $?
+2
+```
+
+This is equivalent to `--to raw0`.
+
 {#--join-output}
 ### `-j`, `--join-output`
 

jaq-fmts/src/lib.rs

@@ -27,6 +27,8 @@ pub enum Format {
     /// When the option `--slurp` is used additionally,
     /// then the whole input is read into a single string.
     Raw,
+    /// Zero-terminated text strings
+    Raw0,
     /// JavaScript Object Notation
     #[default]
     Json,
@@ -61,6 +63,7 @@ impl Format {
         match s {
             "cbor" => Some(Format::Cbor),
             "raw" => Some(Format::Raw),
+            "raw0" => Some(Format::Raw0),
             "json" => Some(Format::Json),
             "toml" => Some(Format::Toml),
             "xml" => Some(Format::Xml),

jaq-fmts/src/read/formats.rs

@@ -3,34 +3,34 @@ use crate::{invalid_data, map_invalid_data, Format};
 use bytes::Bytes;
 use jaq_core::box_iter::{box_once, BoxIter};
 use jaq_json::{Tag, Val};
-use std::io::{self, BufRead, Read};
+use std::io::{self, Read};
 
 type Vals<'a> = BoxIter<'a, io::Result<Val>>;
 
 /// Read input to string for certain formats.
 ///
-/// This has to be synchronised with [`from_bufread`].
+/// This has to be synchronised with [`read`].
 pub fn read_string(fmt: Format, read: impl Read) -> Result<String> {
     use Format::*;
     match fmt {
-        Raw | Json | Cbor => Ok(String::new()),
+        Raw | Raw0 | Json | Cbor => Ok(String::new()),
         Toml | Xml | Yaml => io::read_to_string(read),
     }
 }
 
 /// Convert bytes to string for certain formats.
 ///
-/// This has to be synchronised with [`from_bytes`].
+/// This has to be synchronised with [`parse`].
 pub fn bytes_str(fmt: Format, bytes: &[u8]) -> Result<&str> {
     use Format::*;
     Ok(match fmt {
-        Raw | Json | Cbor => "",
+        Raw | Raw0 | Json | Cbor => "",
         Toml | Xml | Yaml => core::str::from_utf8(bytes).map_err(invalid_data)?,
     })
 }
 
-/// Read value from [`BufRead`] or [`&str`], depending on format.
-pub fn from_bufread<'a>(fmt: Format, read: impl BufRead + 'a, s: &'a str, slurp: bool) -> Vals<'a> {
+/// Read values from [`io::BufRead`] or [`&str`], depending on format.
+pub fn read<'a>(fmt: Format, read: impl io::BufRead + 'a, s: &'a str, slurp: bool) -> Vals<'a> {
     use bstr::io::BufReadExt;
     let mut read = read;
     match fmt {
@@ -40,6 +40,7 @@ pub fn from_bufread<'a>(fmt: Format, read: impl BufRead + 'a, s: &'a str, slurp:
             box_once(result.map(|_| Val::utf8_str(buf)))
         }
         Format::Raw => Box::new(read.byte_lines().map(|r| r.map(Val::utf8_str))),
+        Format::Raw0 => collect_if(slurp, read.byte_records(0).map(|r| r.map(Val::utf8_str))),
         Format::Cbor => collect_if(slurp, cbor::read_many(read)),
         Format::Json => collect_if(slurp, json::read_many(read)),
         Format::Toml => box_once(toml::parse(s).map_err(invalid_data)),
@@ -48,16 +49,17 @@ pub fn from_bufread<'a>(fmt: Format, read: impl BufRead + 'a, s: &'a str, slurp:
     }
 }
 
-/// Parse value from file or `s`, depending on format.
-pub fn from_bytes<'a>(fmt: Format, bytes: &'a Bytes, s: &'a str, slurp: bool) -> Vals<'a> {
+/// Parse values from [`Bytes`] or [`&str`], depending on format.
+pub fn parse<'a>(fmt: Format, bytes: &'a Bytes, s: &'a str, slurp: bool) -> Vals<'a> {
     use bstr::ByteSlice;
+    let nul_sep = |s: &'a [u8]| s.strip_suffix(b"\0").unwrap_or(bytes).split_str("\0");
+    let slice_to_str = |s| Ok(Val::Str(bytes.slice_ref(s), Tag::Utf8));
     match fmt {
         Format::Raw if slurp => box_once(Ok(Val::Str(bytes.clone(), Tag::Utf8))),
-        Format::Raw => Box::new(
-            ByteSlice::lines(&**bytes).map(|line| Ok(Val::Str(bytes.slice_ref(line), Tag::Utf8))),
-        ),
+        Format::Raw => Box::new(bytes.lines().map(slice_to_str)),
+        Format::Raw0 => collect_if(slurp, nul_sep(bytes).map(slice_to_str)),
         Format::Json => collect_if(slurp, json::parse_many(bytes).map(map_invalid_data)),
         Format::Cbor => collect_if(slurp, cbor::parse_many(bytes).map(map_invalid_data)),
-        Format::Toml | Format::Xml | Format::Yaml => from_bufread(fmt, &[][..], s, slurp),
+        Format::Toml | Format::Xml | Format::Yaml => read(fmt, &[][..], s, slurp),
     }
 }

jaq-fmts/src/write/formats.rs

@@ -20,30 +20,32 @@ pub fn write(w: &mut dyn Write, writer: &Writer, val: &Val) -> Result {
     }
 
     match (val, format) {
-        (Val::Str(b, _), Format::Raw) => w.write_all(b)?,
+        (Val::Str(b, _), Format::Raw0) if b.contains(&b'\0') => {
+            let nul_err = "cannot dump a string containing NUL with `--to raw0` or `--raw-output0`";
+            return Err(io::Error::new(io::ErrorKind::InvalidData, nul_err));
+        }
+        (Val::Str(b, _), Format::Raw | Format::Raw0) => w.write_all(b)?,
         (_, Format::Cbor) => cbor::write(w, val)?,
-        (_, Format::Json | Format::Raw) => jaq_json::write::write(w, pp, 0, val)?,
+        (_, Format::Json | Format::Raw | Format::Raw0) => jaq_json::write::write(w, pp, 0, val)?,
         (_, Format::Yaml) => yaml::write(w, pp, 0, val)?,
         (_, Format::Toml) => write!(w, "{}", map_err_to_string(toml::Toml::try_from(val))?)?,
         (_, Format::Xml) => map_err_to_string(xml::Xml::try_from(val))?.write(w)?,
     };
 
-    if match format {
-        Format::Cbor => false,
-        Format::Yaml => true,
-        _ => !join,
-    } {
-        // this flushes output, because stdout is line-buffered in Rust
-        writeln!(w)?
-    };
-
-    // when running `jaq -jn '"prompt> " | (., input)'`,
-    // this flush is necessary to make "prompt> " appear first
-    w.flush()?;
+    w.write_all(match format {
+        Format::Cbor => b"",
+        Format::Raw0 => b"\0",
+        Format::Yaml => b"\n",
+        _ if *join => b"",
+        _ => b"\n",
+    })?;
 
     if yaml_doc {
         // end of YAML document
-        writeln!(w, "...")?
+        writeln!(w, "...")?;
     }
-    Ok(())
+
+    // when running `jaq -jn '"prompt> " | (., input)'`,
+    // this flush is necessary to make "prompt> " appear first
+    w.flush()
 }

jaq/src/cli.rs

@@ -91,11 +91,13 @@ impl Cli {
             "from" => self.from = Some(parse_format("--from", args)?),
             "null-input" => self.short('n', args)?,
             "raw-input" => self.short('R', args)?,
+            "raw-input0" => self.from = Some(Format::Raw0),
             "slurp" => self.short('s', args)?,
 
             "to" => self.to = Some(parse_format("--to", args)?),
             "compact-output" => self.short('c', args)?,
             "raw-output" => self.short('r', args)?,
+            "raw-output0" => self.to = Some(Format::Raw0),
             "join-output" => self.short('j', args)?,
             "in-place" => self.short('i', args)?,
             "sort-keys" => self.short('S', args)?,

jaq/src/help.txt

@@ -9,12 +9,14 @@ Arguments:
 Input options:
   -n, --null-input          Use null as single input value
   -R, --raw-input           Read lines of the input as sequence of strings
+      --raw-input0          Read input like `-R`, splitting by NUL instead of newlines
   -s, --slurp               Read all input values into one array (per file)
       --from <FORMAT>       Read input in given format, e.g. yaml
 
 Output options:
   -c, --compact-output      Print JSON compactly, omitting whitespace
   -r, --raw-output          Write strings without escaping them with quotes
+      --raw-output0         Write output like `-r` and print NUL after each output
   -j, --join-output         Do not print a newline after each value
   -i, --in-place            Overwrite input file with its output
   -S, --sort-keys           Print objects sorted by their keys

jaq/src/main.rs

@@ -129,7 +129,7 @@ fn real_main(cli: &Cli) -> Result<ExitCode, Error> {
     let last = if cli.files.is_empty() {
         let format = unwrap_or_json(cli.from);
         let s = read::read_string(format, io::stdin().lock())?;
-        let inputs = read::from_bufread(format, io::stdin().lock(), &s, cli.slurp);
+        let inputs = read::read(format, io::stdin().lock(), &s, cli.slurp);
         with_stdout(|out| run(runner, &filter, vars, inputs, |v| write(out, writer, &v)))?
     } else {
         let mut last = None;
@@ -139,7 +139,7 @@ fn real_main(cli: &Cli) -> Result<ExitCode, Error> {
                 .map_err(|e| Error::Io(Some(path.display().to_string()), e))?;
             let format = unwrap_or_json(cli.from.or_else(|| Format::determine(path)));
             let s = read::bytes_str(format, &bytes)?;
-            let inputs = read::from_bytes(format, &bytes, s, cli.slurp);
+            let inputs = read::parse(format, &bytes, s, cli.slurp);
 
             if cli.in_place {
                 // create a temporary file where output is written to