Source code

001/*
002 * Licensed to the Apache Software Foundation (ASF) under one or more
003 * contributor license agreements.  See the NOTICE file distributed with
004 * this work for additional information regarding copyright ownership.
005 * The ASF licenses this file to You under the Apache License, Version 2.0
006 * (the "License"); you may not use this file except in compliance with
007 * the License.  You may obtain a copy of the License at
008 *
009 *      http://www.apache.org/licenses/LICENSE-2.0
010 *
011 * Unless required by applicable law or agreed to in writing, software
012 * distributed under the License is distributed on an "AS IS" BASIS,
013 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
014 * See the License for the specific language governing permissions and
015 * limitations under the License.
016 */
017package org.apache.juneau.marshall.hjson;
018
019import static org.apache.juneau.commons.utils.Shorts.*;
020import static org.apache.juneau.commons.utils.StringUtils.*;
021
022import java.io.*;
023import java.util.Locale;
024
025/**
026 * Tokenizer for Hjson format.
027 *
028 * <p>
029 * Handles quoted strings, quoteless strings, multiline strings, comments, numbers, booleans, null,
030 * and structural tokens.
031 *
032 * <h5 class='section'>See Also:</h5><ul>
033 *    <li class='link'><a class="doclink" href="https://hjson.github.io/syntax.html">Hjson Specification</a>
034 * </ul>
035 */
036@SuppressWarnings({
037   "java:S135", // Multiple break/continue necessary for tokenizer state machine loops
038   "java:S3776", // Cognitive complexity acceptable for Hjson grammar
039   "resource" // Reader field wraps the caller's reader (BufferedReader/PushbackReader hold only in-memory buffers); the underlying reader is caller-owned.
040})
041public class HjsonTokenizer {
042
043   /** Token types produced by the tokenizer. */
044   public enum TokenType {
045      /** Double or single-quoted string. */
046      STRING,
047      /** Quoteless string (to end of line). */
048      QUOTELESS,
049      /** Multiline string ('''...'''). */
050      MULTILINE,
051      /** Number. */
052      NUMBER,
053      /** Boolean true. */
054      TRUE,
055      /** Boolean false. */
056      FALSE,
057      /** Null value. */
058      NULL,
059      /** Left brace {. */
060      LBRACE,
061      /** Right brace }. */
062      RBRACE,
063      /** Left bracket [. */
064      LBRACKET,
065      /** Right bracket ]. */
066      RBRACKET,
067      /** Colon :. */
068      COLON,
069      /** Comma ,. */
070      COMMA,
071      /** Significant newline (member separator). */
072      NEWLINE,
073      /** End of input. */
074      EOF
075   }
076
077   /**
078    * A token with optional string or number value.
079    *
080    * @param type Token type.
081    * @param stringValue String value for STRING, QUOTELESS, MULTILINE tokens; <jk>null</jk> otherwise.
082    * @param numberValue Number value for NUMBER tokens; <jk>null</jk> otherwise.
083    */
084   public record Token(TokenType type, String stringValue, Number numberValue) {
085      /**
086       * Creates a structural or literal token with no value.
087       *
088       * @param type Token type.
089       * @return A new token.
090       */
091      public static Token of(TokenType type) {
092         return new Token(type, null, null);
093      }
094      /**
095       * Creates a quoted-string token.
096       *
097       * @param value The string value.
098       * @return A new token.
099       */
100      public static Token string(String value) {
101         return new Token(TokenType.STRING, value, null);
102      }
103      /**
104       * Creates a quoteless-string token.
105       *
106       * @param value The string value.
107       * @return A new token.
108       */
109      public static Token quoteless(String value) {
110         return new Token(TokenType.QUOTELESS, value, null);
111      }
112      /**
113       * Creates a multiline-string token.
114       *
115       * @param value The string value.
116       * @return A new token.
117       */
118      public static Token multiline(String value) {
119         return new Token(TokenType.MULTILINE, value, null);
120      }
121      /**
122       * Creates a number token.
123       *
124       * @param value The number value.
125       * @return A new token.
126       */
127      public static Token number(Number value) {
128         return new Token(TokenType.NUMBER, null, value);
129      }
130   }
131
132   private final PushbackReader reader;
133   private Token peeked;
134   private int line = 1;
135   private int column = 1;
136
137   /**
138    * Constructor.
139    *
140    * @param reader The reader to tokenize.
141    *    <br>Must not be <jk>null</jk>.
142    */
143   public HjsonTokenizer(Reader reader) {
144      var in = reader instanceof BufferedReader reader2 ? reader2 : new BufferedReader(reader);
145      this.reader = new PushbackReader(in, 4);
146   }
147
148   /**
149    * Returns the current line number (1-based).
150    *
151    * @return The line number.
152    */
153   public int getLine() {
154      return line;
155   }
156
157   /**
158    * Returns the current column number (1-based).
159    *
160    * @return The column number.
161    */
162   public int getColumn() {
163      return column;
164   }
165
166   /**
167    * Peeks at the next token without consuming it.
168    *
169    * @return The next token.
170    * @throws IOException If a read error occurs.
171    */
172   public Token peek() throws IOException {
173      if (peeked == null)
174         peeked = readToken();
175      return peeked;
176   }
177
178   /**
179    * Reads and consumes the next token.
180    *
181    * @return The next token.
182    * @throws IOException If a read error occurs.
183    */
184   public Token read() throws IOException {
185      if (peeked != null) {
186         var t = peeked;
187         peeked = null;
188         return t;
189      }
190      return readToken();
191   }
192
193   private int readChar() throws IOException {
194      var c = reader.read();
195      if (c == '\n' || (c == '\r' && peekChar() != '\n')) {
196         line++;
197         column = 1;
198      } else if (c >= 0) {
199         column++;
200      }
201      return c;
202   }
203
204   private int peekChar() throws IOException {
205      var c = reader.read();
206      if (c >= 0)
207         reader.unread(c);
208      return c;
209   }
210
211   private void unread(int c) throws IOException {
212      if (c >= 0) {
213         reader.unread(c);
214         column--;
215         if (c == '\n')
216            line--;
217      }
218   }
219
220   /**
221    * Skips whitespace and comments.
222    *
223    * @throws IOException If a read error occurs.
224    */
225   public void skipWhitespaceAndComments() throws IOException {
226      while (true) {
227         var c = readChar();
228         if (c < 0)
229            return;
230         if (c == ' ' || c == '\t' || c == '\r' || c == '\n')
231            continue;
232         if (c == '#') {
233            skipLineComment();
234            continue;
235         }
236         if (c == '/' && peekChar() == '/') {
237            readChar();
238            skipLineComment();
239            continue;
240         }
241         if (c == '/' && peekChar() == '*') {
242            readChar();
243            skipBlockComment();
244            continue;
245         }
246         unread(c);
247         return;
248      }
249   }
250
251   private void skipLineComment() throws IOException {
252      var c = readChar();
253      while (c >= 0 && c != '\n' && c != '\r')
254         c = readChar();
255      if (c == '\r' && peekChar() == '\n')
256         readChar();
257   }
258
259   private void skipBlockComment() throws IOException {
260      var c = readChar();
261      while (c >= 0) {
262         if (c == '*' && peekChar() == '/') {
263            readChar();
264            return;
265         }
266         c = readChar();
267      }
268   }
269
270   private Token readToken() throws IOException {
271      skipWhitespaceAndComments();
272      var c = readChar();
273      if (c < 0)
274         return Token.of(TokenType.EOF);
275
276      if (c == '{') return Token.of(TokenType.LBRACE);
277      if (c == '}') return Token.of(TokenType.RBRACE);
278      if (c == '[') return Token.of(TokenType.LBRACKET);
279      if (c == ']') return Token.of(TokenType.RBRACKET);
280      if (c == ':') return Token.of(TokenType.COLON);
281      if (c == ',') return Token.of(TokenType.COMMA);
282      if (c == '\n' || c == '\r') {
283         if (c == '\r' && peekChar() == '\n')
284            readChar();
285         return Token.of(TokenType.NEWLINE);
286      }
287
288      if (c == '\'' && peekChar() == '\'' && peekChar(2) == '\'') {
289         readChar();
290         readChar();
291         var indentBaseline = getColumn() - 3;
292         return Token.multiline(readMultilineString(indentBaseline));
293      }
294      if (c == '"' || c == '\'') {
295         unread(c);
296         return Token.string(readQuotedString());
297      }
298
299      unread(c);
300
301      var sb = new StringBuilder();
302      var next = readChar();
303      while (next >= 0 && next != '\n' && next != '\r' && next != ',' && next != ':' && next != '{' && next != '}' && next != '[' && next != ']') {
304         sb.append((char) next);
305         next = readChar();
306      }
307      if (next >= 0)
308         unread(next);
309
310      var raw = sb.toString().trim();
311
312      if (raw.isEmpty())
313         return readToken();
314
315      if (eq(raw, "true")) return Token.of(TokenType.TRUE);
316      if (eq(raw, "false")) return Token.of(TokenType.FALSE);
317      if (eq(raw, "null")) return Token.of(TokenType.NULL);
318
319      // Whole-string JSON-number match (leading-zero-restricted grammar); span equals the full token on a match.
320      if (matchNumberPrefix(raw, false) == raw.length()) {
321         var numStr = raw;
322         try {
323            if (numStr.contains(".") || numStr.toLowerCase(Locale.ROOT).contains("e"))
324               return Token.number(Double.parseDouble(numStr));
325            return parseIntegerOrLongToken(numStr);
326         } catch (@SuppressWarnings("unused") NumberFormatException e) {
327            // Number too large for Long or other parse failure; fall through to quoteless
328         }
329      }
330
331      return Token.quoteless(raw);
332   }
333
334   private static Token parseIntegerOrLongToken(String numStr) {
335      try {
336         return Token.number(Integer.parseInt(numStr));
337      } catch (@SuppressWarnings("unused") NumberFormatException e) {
338         return Token.number(Long.parseLong(numStr));
339      }
340   }
341
342   private int peekChar(int offset) throws IOException {
343      var buf = new int[offset];
344      for (var i = 0; i < offset; i++)
345         buf[i] = readChar();
346      for (var i = offset - 1; i >= 0; i--)
347         unread(buf[i]);
348      return offset > 0 ? buf[offset - 1] : -1;
349   }
350
351   /**
352    * Reads a quoted string ("..." or '...') with JSON-style escaping.
353    *
354    * @return The string value.
355    * @throws IOException If a read error occurs.
356    */
357   public String readQuotedString() throws IOException {
358      var quote = readChar();
359      if (quote != '"' && quote != '\'')
360         return "";
361      var sb = new StringBuilder();
362      var c = readChar();
363      while (c >= 0 && c != quote) {
364         if (c == '\\') {
365            c = readChar();
366            if (c >= 0) {
367               if (c == 'u')
368                  sb.append(readUnicodeEscape());
369               else
370                  sb.append(unescape(c));
371            }
372         } else {
373            sb.append((char) c);
374         }
375         c = readChar();
376      }
377      return sb.toString();
378   }
379
380   private static char unescape(int c) {
381      return switch (c) {
382         case '"' -> '"';
383         case '\'' -> '\'';
384         case '\\' -> '\\';
385         case '/' -> '/';
386         case 'b' -> '\b';
387         case 'f' -> '\f';
388         case 'n' -> '\n';
389         case 'r' -> '\r';
390         case 't' -> '\t';
391         default -> (char) c;
392      };
393   }
394
395   private char readUnicodeEscape() throws IOException {
396      var hex = new StringBuilder(4);
397      for (var i = 0; i < 4; i++) {
398         var c = readChar();
399         if (c < 0 || !isHexDigit(c))
400            throw ioex("Invalid \\u escape sequence at line %s, column %s", line, column);
401         hex.append((char) c);
402      }
403      return (char) Integer.parseInt(hex.toString(), 16);
404   }
405
406   private static boolean isHexDigit(int c) {
407      return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F');
408   }
409
410   /**
411    * Reads a quoteless string (to end of line, trimmed).
412    *
413    * @return The string value.
414    * @throws IOException If a read error occurs.
415    */
416   public String readQuotelessString() throws IOException {
417      var sb = new StringBuilder();
418      var c = readChar();
419      while (c >= 0 && c != '\n' && c != '\r') {
420         sb.append((char) c);
421         c = readChar();
422      }
423      if (c >= 0)
424         unread(c);
425      return sb.toString().trim();
426   }
427
428   /**
429    * Reads a multiline string ('''...''').
430    * Caller must have already consumed the opening '''.
431    *
432    * @param indentBaseline The 1-based column of the first quote in '''; used to strip
433    *    leading indentation from subsequent lines per Hjson spec.
434    * @return The string value (with \n line separators, last newline ignored per spec).
435    * @throws IOException If a read error occurs.
436    */
437   public String readMultilineString(int indentBaseline) throws IOException {
438      var sb = new StringBuilder();
439      var quoteCount = 0;
440      var c = readChar();
441      while (c >= 0) {
442         if (c == '\'') {
443            quoteCount++;
444            if (quoteCount == 3) {
445               var s = sb.toString();
446               if (s.endsWith("\n"))
447                  s = s.substring(0, s.length() - 1);
448               s = s.replace("\r\n", "\n").replace("\r", "\n");
449               return stripMultilineIndent(s, indentBaseline);
450            }
451         } else {
452            if (quoteCount == 1)
453               sb.append("'");
454            else if (quoteCount == 2)
455               sb.append("''");
456            quoteCount = 0;
457            if (c == '\r' && peekChar() == '\n') {
458               readChar();
459               sb.append('\n');
460            } else if (c != '\r') {
461               sb.append((char) c);
462            }
463         }
464         c = readChar();
465      }
466      return stripMultilineIndent(sb.toString(), indentBaseline);
467   }
468
469   private static String stripMultilineIndent(String s, int indentBaseline) {
470      if (indentBaseline <= 0 || s.isEmpty())
471         return s;
472      var stripCount = indentBaseline - 1;
473      if (stripCount <= 0)
474         return s;
475      var lines = s.split("\n", -1);
476      var result = new StringBuilder();
477      for (var i = 0; i < lines.length; i++) {
478         if (i > 0)
479            result.append('\n');
480         var line = lines[i];
481         var j = 0;
482         while (j < stripCount && j < line.length() && (line.charAt(j) == ' ' || line.charAt(j) == '\t'))
483            j++;
484         result.append(line.substring(j));
485      }
486      return result.toString();
487   }
488}