diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..21c125c --- /dev/null +++ b/.gitattributes @@ -0,0 +1,11 @@ +# SPDX-FileCopyrightText: 2024 Justin Myers for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense + +.py text eol=lf +.rst text eol=lf +.txt text eol=lf +.yaml text eol=lf +.toml text eol=lf +.license text eol=lf +.md text eol=lf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e5fccc..ff19dde 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,42 +1,21 @@ -# SPDX-FileCopyrightText: 2020 Diego Elio Pettenò +# SPDX-FileCopyrightText: 2024 Justin Myers for Adafruit Industries # # SPDX-License-Identifier: Unlicense repos: - - repo: https://github.com/python/black - rev: 22.3.0 - hooks: - - id: black - - repo: https://github.com/fsfe/reuse-tool - rev: v0.14.0 - hooks: - - id: reuse - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.2.0 + rev: v4.5.0 hooks: - id: check-yaml - id: end-of-file-fixer - id: trailing-whitespace - - repo: https://github.com/pycqa/pylint - rev: v2.15.5 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.3.4 hooks: - - id: pylint - name: pylint (library code) - types: [python] - args: - - --disable=consider-using-f-string - exclude: "^(docs/|examples/|tests/|setup.py$)" - - id: pylint - name: pylint (example code) - description: Run pylint rules on "examples/*.py" files - types: [python] - files: "^examples/" - args: - - --disable=missing-docstring,invalid-name,consider-using-f-string,duplicate-code - - id: pylint - name: pylint (test code) - description: Run pylint rules on "tests/*.py" files - types: [python] - files: "^tests/" - args: - - --disable=missing-docstring,consider-using-f-string,duplicate-code + - id: ruff-format + - id: ruff + args: ["--fix"] + - repo: https://github.com/fsfe/reuse-tool + rev: v3.0.1 + hooks: + - id: reuse diff --git a/.pylintrc b/.pylintrc deleted file mode 100644 index f945e92..0000000 --- a/.pylintrc +++ /dev/null @@ -1,399 +0,0 @@ -# SPDX-FileCopyrightText: 2017 Scott Shawcroft, written for Adafruit Industries -# -# SPDX-License-Identifier: Unlicense - -[MASTER] - -# A comma-separated list of package or module names from where C extensions may -# be loaded. Extensions are loading into the active Python interpreter and may -# run arbitrary code -extension-pkg-whitelist= - -# Add files or directories to the ignore-list. They should be base names, not -# paths. -ignore=CVS - -# Add files or directories matching the regex patterns to the ignore-list. The -# regex matches against base names, not paths. -ignore-patterns= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Use multiple processes to speed up Pylint. -jobs=1 - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins=pylint.extensions.no_self_use - -# Pickle collected data for later comparisons. -persistent=yes - -# Specify a configuration file. -#rcfile= - -# Allow loading of arbitrary C extensions. Extensions are imported into the -# active Python interpreter and may run arbitrary code. -unsafe-load-any-extension=no - - -[MESSAGES CONTROL] - -# Only show warnings with the listed confidence levels. Leave empty to show -# all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED -confidence= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -# disable=import-error,raw-checker-failed,bad-inline-option,locally-disabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,deprecated-str-translate-call -disable=raw-checker-failed,bad-inline-option,locally-disabled,file-ignored,suppressed-message,useless-suppression,deprecated-pragma,import-error,pointless-string-statement,unspecified-encoding - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time (only on the command line, not in the configuration file where -# it should appear only once). See also the "--disable" option for examples. -enable= - - -[REPORTS] - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Template used to display messages. This is a python new-style format string -# used to format the message information. See doc for all details -#msg-template= - -# Set the output format. Available formats are text, parseable, colorized, json -# and msvs (visual studio).You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Tells whether to display a full report or only the messages -reports=no - -# Activate the evaluation score. -score=yes - - -[REFACTORING] - -# Maximum number of nested blocks for function / method body -max-nested-blocks=5 - - -[LOGGING] - -# Logging modules to check that the string format arguments are in logging -# function parameter format -logging-modules=logging - - -[SPELLING] - -# Spelling dictionary name. Available dictionaries: none. To make it working -# install python-enchant package. -spelling-dict= - -# List of comma separated words that should not be checked. -spelling-ignore-words= - -# A path to a file that contains private dictionary; one word per line. -spelling-private-dict-file= - -# Tells whether to store unknown words to indicated private dictionary in -# --spelling-private-dict-file option instead of raising a message. -spelling-store-unknown-words=no - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -# notes=FIXME,XXX,TODO -notes=FIXME,XXX - - -[TYPECHECK] - -# List of decorators that produce context managers, such as -# contextlib.contextmanager. Add to this list to register other decorators that -# produce valid context managers. -contextmanager-decorators=contextlib.contextmanager - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E1101 when accessed. Python regular -# expressions are accepted. -generated-members= - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# This flag controls whether pylint should warn about no-member and similar -# checks whenever an opaque object is returned when inferring. The inference -# can return multiple potential results while evaluating a Python object, but -# some branches might not be evaluated, which results in partial inference. In -# that case, it might be useful to still emit no-member and other checks for -# the rest of the inferred objects. -ignore-on-opaque-inference=yes - -# List of class names for which member attributes should not be checked (useful -# for classes with dynamically set attributes). This supports the use of -# qualified names. -ignored-classes=optparse.Values,thread._local,_thread._local - -# List of module names for which member attributes should not be checked -# (useful for modules/projects where namespaces are manipulated during runtime -# and thus existing member attributes cannot be deduced by static analysis. It -# supports qualified module names, as well as Unix pattern matching. -ignored-modules=board - -# Show a hint with possible names when a member name was not found. The aspect -# of finding the hint is based on edit distance. -missing-member-hint=yes - -# The minimum edit distance a name should have in order to be considered a -# similar match for a missing member name. -missing-member-hint-distance=1 - -# The total number of similar names that should be taken in consideration when -# showing a hint for a missing member. -missing-member-max-choices=1 - - -[VARIABLES] - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - -# Tells whether unused global variables should be treated as a violation. -allow-global-unused-variables=yes - -# List of strings which can identify a callback function by name. A callback -# name must start or end with one of those strings. -callbacks=cb_,_cb - -# A regular expression matching the name of dummy variables (i.e. expectedly -# not used). -dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.*|^ignored_|^unused_ - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# List of qualified module names which can have objects that can redefine -# builtins. -redefining-builtins-modules=six.moves,future.builtins - - -[FORMAT] - -# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. -# expected-line-ending-format= -expected-line-ending-format=LF - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )??$ - -# Number of spaces of indent required inside a hanging or continued line. -indent-after-paren=4 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - -# Maximum number of characters on a single line. -max-line-length=100 - -# Maximum number of lines in a module -max-module-lines=1000 - -# Allow the body of a class to be on the same line as the declaration if body -# contains single statement. -single-line-class-stmt=no - -# Allow the body of an if to be on the same line as the test if there is no -# else. -single-line-if-stmt=no - - -[SIMILARITIES] - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=yes - -# Minimum lines number of a similarity. -min-similarity-lines=12 - - -[BASIC] - -# Regular expression matching correct argument names -argument-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct attribute names -attr-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Regular expression matching correct class attribute names -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression matching correct class names -# class-rgx=[A-Z_][a-zA-Z0-9]+$ -class-rgx=[A-Z_][a-zA-Z0-9_]+$ - -# Regular expression matching correct constant names -const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - -# Regular expression matching correct function names -function-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Good variable names which should always be accepted, separated by a comma -# good-names=i,j,k,ex,Run,_ -good-names=r,g,b,w,i,j,k,n,x,y,z,ex,ok,Run,_ - -# Include a hint for the correct naming format with invalid-name -include-naming-hint=no - -# Regular expression matching correct inline iteration names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Regular expression matching correct method names -method-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - -# Regular expression matching correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Colon-delimited sets of names that determine each other's naming style when -# the name regexes allow several styles. -name-group= - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=^_ - -# List of decorators that produce properties, such as abc.abstractproperty. Add -# to this list to register other decorators that produce valid properties. -property-classes=abc.abstractproperty - -# Regular expression matching correct variable names -variable-rgx=(([a-z][a-z0-9_]{2,30})|(_[a-z0-9_]*))$ - - -[IMPORTS] - -# Allow wildcard imports from modules that define __all__. -allow-wildcard-with-all=no - -# Analyse import fallback blocks. This can be used to support both Python 2 and -# 3 compatible code, which means that the block might have code that exists -# only in one or another interpreter, leading to false positives when analysed. -analyse-fallback-blocks=no - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=optparse,tkinter.tix - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - -# Force import order to recognize a module as part of the standard -# compatibility libraries. -known-standard-library= - -# Force import order to recognize a module as part of a third party library. -known-third-party=enchant - - -[CLASSES] - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of member names, which should be excluded from the protected access -# warning. -exclude-protected=_asdict,_fields,_replace,_source,_make - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Maximum number of attributes for a class (see R0902). -# max-attributes=7 -max-attributes=11 - -# Maximum number of boolean expressions in a if statement -max-bool-expr=5 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of statements in function / method body -max-statements=50 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=1 - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=builtins.Exception diff --git a/.readthedocs.yaml b/.readthedocs.yaml index fe4faae..ee38fa0 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -12,7 +12,7 @@ sphinx: configuration: docs/conf.py build: - os: ubuntu-20.04 + os: ubuntu-lts-latest tools: python: "3" diff --git a/README.rst b/README.rst index 1605ba1..15a249c 100644 --- a/README.rst +++ b/README.rst @@ -17,9 +17,9 @@ Introduction :alt: Build Status -.. image:: https://img.shields.io/badge/code%20style-black-000000.svg - :target: https://github.com/psf/black - :alt: Code Style: Black +.. image:: https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json + :target: https://github.com/astral-sh/ruff + :alt: Code Style: Ruff This library is a reimplementation and subset of `json_stream `_. It enables reading JSON data from a stream rather that loading it all into memory at once. The interface works like lists and dictionaries that are usually returned from ``json.load()`` but require in-order access. Out of order accesses will lead to missing keys and list entries. diff --git a/adafruit_json_stream.py b/adafruit_json_stream.py index 78831a9..fe863d1 100644 --- a/adafruit_json_stream.py +++ b/adafruit_json_stream.py @@ -26,6 +26,7 @@ def __init__(self, data_iter): self.data_iter = data_iter self.i = 0 self.chunk = b"" + self.last_char = None def read(self): """Read the next character from the stream.""" @@ -39,30 +40,61 @@ def read(self): self.i += 1 return char - def fast_forward(self, closer): - """Read through the stream until the character is ``closer``, ``]`` + def fast_forward(self, closer, *, return_object=False): + """ + Read through the stream until the character is ``closer``, ``]`` (ending a list) or ``}`` (ending an object.) Intermediate lists and - objects are skipped.""" + objects are skipped. + + :param str closer: the character to read until + :param bool return_object: read until the closer, + and then parse the data and return as an object + """ + closer = ord(closer) close_stack = [closer] count = 0 + + buffer = None + if return_object: + buffer = bytearray(32) + # ] = 93, [ = 91 + # } = 125, { = 123 + buffer[0] = closer - 2 + + ignore_next = False while close_stack: char = self.read() count += 1 - if char == close_stack[-1]: + if buffer: + if count == len(buffer): + new_buffer = bytearray(len(buffer) + 32) + new_buffer[: len(buffer)] = buffer + buffer = new_buffer + buffer[count] = char + if ignore_next: + # that character was escaped, skip it + ignore_next = False + elif char == close_stack[-1]: close_stack.pop() + elif char == ord("\\") and close_stack[-1] == ord('"'): + # if backslash, ignore the next character + ignore_next = True elif char == ord('"'): close_stack.append(ord('"')) elif close_stack[-1] == ord('"'): # in a string so ignore [] and {} pass - elif char in (ord("}"), ord("]")): + elif char in {ord("}"), ord("]")}: # Mismatched list or object means we're done and already past the last comma. return True elif char == ord("{"): close_stack.append(ord("}")) elif char == ord("["): close_stack.append(ord("]")) + if buffer: + value_string = bytes(memoryview(buffer)[: count + 1]).decode("utf-8") + return json.loads(value_string) return False def next_value(self, endswith=None): @@ -71,56 +103,86 @@ def next_value(self, endswith=None): if isinstance(endswith, str): endswith = ord(endswith) in_string = False + ignore_next = False while True: try: char = self.read() except EOFError: char = endswith - if not in_string and (char == endswith or char in (ord("]"), ord("}"))): - if len(buf) == 0: - return None - value_string = bytes(buf).decode("utf-8") - # print(f"{repr(value_string)}, {endswith=}") - return json.loads(value_string) - if char == ord("{"): - return TransientObject(self) - if char == ord("["): - return TransientList(self) + in_string = False + ignore_next = False if not in_string: - in_string = char == ord('"') - else: - in_string = char != ord('"') - buf.append(char) + # end character or object/list end + if char == endswith or char in {ord("]"), ord("}")}: + self.last_char = char + if len(buf) == 0: + return None + value_string = bytes(buf).decode("utf-8") + return json.loads(value_string) + # string or sub object + if char == ord("{"): + return TransientObject(self) + if char == ord("["): + return TransientList(self) + # start a string + if char == ord('"'): + in_string = True + # skipping any closing or opening character if in a string + # also skipping escaped characters (like quotes in string) + elif ignore_next: + ignore_next = False + elif char == ord("\\"): + ignore_next = True + elif char == ord('"'): + in_string = False -class Transient: # pylint: disable=too-few-public-methods - """Transient object representing a JSON object.""" - - # This is helpful for checking that something is a TransientList or TransientObject. + buf.append(char) -class TransientList(Transient): - """Transient object that acts like a list through the stream.""" +class Transient: + """Transient object representing a JSON object.""" def __init__(self, stream): + self.active_child = None self.data = stream self.done = False - self.active_child = None + self.has_read = False + self.finish_char = "" def finish(self): - """Consume all of the characters for this list from the stream.""" + """Consume all of the characters for this container from the stream.""" if not self.done: if self.active_child: self.active_child.finish() self.active_child = None - self.data.fast_forward("]") + self.data.fast_forward(self.finish_char) self.done = True + def as_object(self): + """Consume all of the characters for this container from the stream + and return as an object.""" + if self.has_read: + raise BufferError("Object has already been partly read.") + + self.done = True + return self.data.fast_forward(self.finish_char, return_object=True) + + +class TransientList(Transient): + """Transient object that acts like a list through the stream.""" + + def __init__(self, stream): + super().__init__(stream) + self.finish_char = "]" + def __iter__(self): return self def __next__(self): + self.has_read = True + if self.active_child: self.active_child.finish() self.done = self.data.fast_forward(",") @@ -128,6 +190,8 @@ def __next__(self): if self.done: raise StopIteration() next_value = self.data.next_value(",") + if self.data.last_char == ord("]"): + self.done = True if next_value is None: self.done = True raise StopIteration() @@ -140,42 +204,86 @@ class TransientObject(Transient): """Transient object that acts like a dictionary through the stream.""" def __init__(self, stream): - self.data = stream - self.done = False - self.buf = array.array("B") - - self.active_child = None + super().__init__(stream) + self.finish_char = "}" + self.active_key = None def finish(self): - """Consume all of the characters for this object from the stream.""" - if not self.done: - if self.active_child: - self.active_child.finish() - self.active_child = None - self.data.fast_forward("}") - self.done = True + """Consume all of the characters for this container from the stream.""" + if self.active_key and not self.active_child: + self.done = self.data.fast_forward(",") + self.active_key = None + super().finish() def __getitem__(self, key): + if self.active_child and self.active_key == key: + return self.active_child + + self.has_read = True + if self.active_child: self.active_child.finish() self.done = self.data.fast_forward(",") self.active_child = None + self.active_key = None if self.done: - raise KeyError() + raise KeyError(key) - while True: - current_key = self.data.next_value(":") + while not self.done: + if self.active_key: + current_key = self.active_key + self.active_key = None + else: + current_key = self.data.next_value(":") if current_key is None: - # print("object done", self) self.done = True break if current_key == key: next_value = self.data.next_value(",") + if self.data.last_char == ord("}"): + self.done = True if isinstance(next_value, Transient): self.active_child = next_value + self.active_key = key return next_value - self.data.fast_forward(",") - raise KeyError() + self.done = self.data.fast_forward(",") + raise KeyError(key) + + def __iter__(self): + return self + + def _next_key(self): + """Return the next item's key, without consuming the value.""" + if self.active_key: + if self.active_child: + self.active_child.finish() + self.active_child = None + self.done = self.data.fast_forward(",") + self.active_key = None + if self.done: + raise StopIteration() + + self.has_read = True + + current_key = self.data.next_value(":") + if current_key is None: + self.done = True + raise StopIteration() + + self.active_key = current_key + return current_key + + def __next__(self): + return self._next_key() + + def items(self): + """Return iterator in the dictionary’s items ((key, value) pairs).""" + try: + while not self.done: + key = self._next_key() + yield (key, self[key]) + except StopIteration: + return def load(data_iter): diff --git a/docs/api.rst b/docs/api.rst index e949b4e..6f493d3 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -4,5 +4,8 @@ .. If your library file(s) are nested in a directory (e.g. /adafruit_foo/foo.py) .. use this format as the module name: "adafruit_foo.foo" +API Reference +############# + .. automodule:: adafruit_json_stream :members: diff --git a/docs/conf.py b/docs/conf.py index 6c9622c..590a32e 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,12 +1,10 @@ -# -*- coding: utf-8 -*- - # SPDX-FileCopyrightText: 2017 Scott Shawcroft, written for Adafruit Industries # # SPDX-License-Identifier: MIT +import datetime import os import sys -import datetime sys.path.insert(0, os.path.abspath("..")) @@ -53,9 +51,7 @@ creation_year = "2023" current_year = str(datetime.datetime.now().year) year_duration = ( - current_year - if current_year == creation_year - else creation_year + " - " + current_year + current_year if current_year == creation_year else creation_year + " - " + current_year ) copyright = year_duration + " Scott Shawcroft" author = "Scott Shawcroft" diff --git a/examples/json_stream_local_file.py b/examples/json_stream_local_file.py index 443f4f5..214854d 100644 --- a/examples/json_stream_local_file.py +++ b/examples/json_stream_local_file.py @@ -18,7 +18,7 @@ def iter_content(self, chunk_size): yield self.file.read(chunk_size) -f = open(sys.argv[1], "rb") # pylint: disable=consider-using-with +f = open(sys.argv[1], "rb") obj = json_stream.load(FakeResponse(f).iter_content(32)) currently = obj["currently"] diff --git a/examples/json_stream_local_file_advanced.py b/examples/json_stream_local_file_advanced.py new file mode 100644 index 0000000..de34b46 --- /dev/null +++ b/examples/json_stream_local_file_advanced.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: Copyright (c) 2023 Scott Shawcroft for Adafruit Industries +# +# SPDX-License-Identifier: Unlicense + +import sys +import time + +import adafruit_json_stream as json_stream + +# import json_stream + + +class FakeResponse: + def __init__(self, file): + self.file = file + + def iter_content(self, chunk_size): + while True: + yield self.file.read(chunk_size) + + +f = open(sys.argv[1], "rb") +obj = json_stream.load(FakeResponse(f).iter_content(32)) + + +def find_keys(haystack, keys): + """If we don't know the order in which the keys are, + go through all of them and pick the ones we want""" + out = {} + # iterate on the items of an object + for key in haystack: + if key in keys: + # retrieve the value only if needed + value = haystack[key] + # if it's a sub object, get it all + if hasattr(value, "as_object"): + value = value.as_object() + out[key] = value + return out + + +months = [ + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +] + + +def time_to_date(stamp): + tt = time.localtime(stamp) + month = months[tt.tm_mon] + return f"{tt.tm_mday:2d}th of {month}" + + +def ftoc(temp): + return (temp - 32) * 5 / 9 + + +currently = obj["currently"] +print("Currently:") +print(" ", time_to_date(currently["time"])) +print(" ", currently["icon"]) + +# iterate on the content of a list +for i, day in enumerate(obj["daily"]["data"]): + day_items = find_keys(day, ("time", "summary", "temperatureHigh")) + date = time_to_date(day_items["time"]) + print( + f'On {date}: {day_items["summary"]},', + f'Max: {int(day_items["temperatureHigh"])}F', + f'({int(ftoc(day_items["temperatureHigh"]))}C)', + ) + + if i > 4: + break diff --git a/examples/json_stream_simpletest.py b/examples/json_stream_simpletest.py index f298630..093cdd3 100644 --- a/examples/json_stream_simpletest.py +++ b/examples/json_stream_simpletest.py @@ -5,9 +5,11 @@ import ssl import time + import adafruit_requests import socketpool import wifi + import adafruit_json_stream as json_stream pool = socketpool.SocketPool(wifi.radio) diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..73e9efc --- /dev/null +++ b/ruff.toml @@ -0,0 +1,108 @@ +# SPDX-FileCopyrightText: 2024 Tim Cocks for Adafruit Industries +# +# SPDX-License-Identifier: MIT + +target-version = "py38" +line-length = 100 + +[lint] +preview = true +select = ["I", "PL", "UP"] + +extend-select = [ + "D419", # empty-docstring + "E501", # line-too-long + "W291", # trailing-whitespace + "PLC0414", # useless-import-alias + "PLC2401", # non-ascii-name + "PLC2801", # unnecessary-dunder-call + "PLC3002", # unnecessary-direct-lambda-call + "E999", # syntax-error + "PLE0101", # return-in-init + "F706", # return-outside-function + "F704", # yield-outside-function + "PLE0116", # continue-in-finally + "PLE0117", # nonlocal-without-binding + "PLE0241", # duplicate-bases + "PLE0302", # unexpected-special-method-signature + "PLE0604", # invalid-all-object + "PLE0605", # invalid-all-format + "PLE0643", # potential-index-error + "PLE0704", # misplaced-bare-raise + "PLE1141", # dict-iter-missing-items + "PLE1142", # await-outside-async + "PLE1205", # logging-too-many-args + "PLE1206", # logging-too-few-args + "PLE1307", # bad-string-format-type + "PLE1310", # bad-str-strip-call + "PLE1507", # invalid-envvar-value + "PLE2502", # bidirectional-unicode + "PLE2510", # invalid-character-backspace + "PLE2512", # invalid-character-sub + "PLE2513", # invalid-character-esc + "PLE2514", # invalid-character-nul + "PLE2515", # invalid-character-zero-width-space + "PLR0124", # comparison-with-itself + "PLR0202", # no-classmethod-decorator + "PLR0203", # no-staticmethod-decorator + "UP004", # useless-object-inheritance + "PLR0206", # property-with-parameters + "PLR0904", # too-many-public-methods + "PLR0911", # too-many-return-statements + "PLR0912", # too-many-branches + "PLR0913", # too-many-arguments + "PLR0914", # too-many-locals + "PLR0915", # too-many-statements + "PLR0916", # too-many-boolean-expressions + "PLR1702", # too-many-nested-blocks + "PLR1704", # redefined-argument-from-local + "PLR1711", # useless-return + "C416", # unnecessary-comprehension + "PLR1733", # unnecessary-dict-index-lookup + "PLR1736", # unnecessary-list-index-lookup + + # ruff reports this rule is unstable + #"PLR6301", # no-self-use + + "PLW0108", # unnecessary-lambda + "PLW0120", # useless-else-on-loop + "PLW0127", # self-assigning-variable + "PLW0129", # assert-on-string-literal + "B033", # duplicate-value + "PLW0131", # named-expr-without-context + "PLW0245", # super-without-brackets + "PLW0406", # import-self + "PLW0602", # global-variable-not-assigned + "PLW0603", # global-statement + "PLW0604", # global-at-module-level + + # fails on the try: import typing used by libraries + #"F401", # unused-import + + "F841", # unused-variable + "E722", # bare-except + "PLW0711", # binary-op-exception + "PLW1501", # bad-open-mode + "PLW1508", # invalid-envvar-default + "PLW1509", # subprocess-popen-preexec-fn + "PLW2101", # useless-with-lock + "PLW3301", # nested-min-max +] + +ignore = [ + "PLR2004", # magic-value-comparison + "UP030", # format literals + "PLW1514", # unspecified-encoding + "PLR0913", # too-many-arguments + "PLR0915", # too-many-statements + "PLR0917", # too-many-positional-arguments + "PLR0904", # too-many-public-methods + "PLR0912", # too-many-branches + "PLR0916", # too-many-boolean-expressions + "PLR6301", # could-be-static no-self-use + "PLC0415", # import outside toplevel + "PLC2701", # private import +] + +[format] +line-ending = "lf" diff --git a/tests/test_json_stream.py b/tests/test_json_stream.py new file mode 100644 index 0000000..457186b --- /dev/null +++ b/tests/test_json_stream.py @@ -0,0 +1,755 @@ +# SPDX-FileCopyrightText: 2025 Justin Myers +# +# SPDX-License-Identifier: Unlicense + +import json +import math + +import pytest + +import adafruit_json_stream + +# --------------- +# Helpers +# --------------- + + +class BytesChunkIO: + def __init__(self, data=b"", chunk_size=10): + self.chunk_size = chunk_size + self.chunks_read = 0 + self.data = data + self.data_len = len(self.data) + self.position = 0 + + def __iter__(self): + return self + + def __next__(self): + if self.position > self.data_len: + raise StopIteration + + end = self.chunk_size + if self.position + end > self.data_len: + end = self.data_len + chunk = self.data[self.position : self.position + self.chunk_size] + + self.chunks_read += 1 + self.position += self.chunk_size + + return chunk + + def get_chunks_read(self): + return self.chunks_read + + +# --------------- +# Fixtures +# --------------- + + +@pytest.fixture +def dict_with_all_types(): + return """ + { + "_check": "{\\\"a\\\": 1, \\\"b\\\": [2,3]}", + "bool": true, + "dict": {"key": "value"}, + "float": 1.1, + "int": 1, + "list": [1,2,3], + "null": null, + "string": "string" + } + """ + + +@pytest.fixture +def list_with_bad_strings(): + return r""" + [ + "\"}\"", + "{\"a\": 1, \"b\": [2,3]}", + "\"", + "\\\"", + "\\\\\"", + "\\x40\"", + "[[[{{{", + "]]]}}}" + ] + """ + + +@pytest.fixture +def dict_with_bad_strings(): + return r""" + { + "1": "\"}\"", + "2": "{\"a\": 1, \"b\": [2,3]}", + "3": "\"", + "4": "\\\"", + "5": "\\\\\"", + "6": "\\x40\"", + "7": "[[[{{{", + "8": "]]]}}}" + } + """ + + +@pytest.fixture +def list_with_values(): + return """ + [ + 1, + 2, + 3 + ] + """ + + +@pytest.fixture +def dict_with_keys(): + return """ + { + "field_1": 1, + "field_2": 2, + "field_3": 3 + } + """ + + +@pytest.fixture +def dict_with_list_with_single_entries(): + return """ + { + "list_1": [ + { + "dict_id": 1 + }, + { + "dict_id": 2 + }, + { + "dict_id": 3 + }, + { + "dict_id": 4 + } + ] + } + """ + + +@pytest.fixture +def complex_dict(): + return """ + { + "list_1": [ + { + "dict_id": 1, + "dict_name": "one", + "sub_dict": { + "sub_dict_id": 1.1, + "sub_dict_name": "one point one" + }, + "sub_list": [ + "a", + "b", + "c" + ] + }, + { + "dict_id": 2, + "dict_name": "two", + "sub_dict": { + "sub_dict_id": 2.1, + "sub_dict_name": "two point one" + }, + "sub_list": [ + "d", + "e", + "f" + ] + } + ], + "list_2": [ + { + "dict_id": 3, + "dict_name": "three", + "sub_dict": { + "sub_dict_id": 3.1, + "sub_dict_name": "three point one" + }, + "sub_list": [ + "g", + "h", + "i" + ] + }, + { + "dict_id": 4, + "dict_name": "four", + "sub_dict": { + "sub_dict_id": 4.1, + "sub_dict_name": "four point one" + }, + "sub_list": [ + "j", + "k", + "l" + ] + } + ] + } + """ + + +# --------------- +# Tests +# --------------- + + +def test_all_types(dict_with_all_types): + """Test loading a simple dict all data types.""" + + assert json.loads(dict_with_all_types) + + stream = adafruit_json_stream.load(BytesChunkIO(dict_with_all_types.encode())) + + assert stream["bool"] is True + assert stream["dict"]["key"] == "value" + assert stream["float"] == 1.1 + assert stream["int"] == 1 + assert next(stream["list"]) == 1 + assert stream["null"] is None + assert stream["string"] == "string" + + +def test_simple_dict_with_keys(dict_with_keys): + """Test loading a simple dict with keys.""" + + assert json.loads(dict_with_keys) + + stream = adafruit_json_stream.load(BytesChunkIO(dict_with_keys.encode())) + for i in range(1, 4): + assert stream[f"field_{i}"] == i + with pytest.raises(KeyError, match="field_4"): + stream["field_4"] + + +def test_simple_dict_with_grabbing_key_twice_raises(dict_with_keys): + """Test loading a simple dict with keys twice raises.""" + + assert json.loads(dict_with_keys) + + stream = adafruit_json_stream.load(BytesChunkIO(dict_with_keys.encode())) + assert stream["field_1"] == 1 + with pytest.raises(KeyError, match="field_1"): + stream["field_1"] + + +def test_simple_dict_with_keys_middle_key(dict_with_keys): + """Test loading a simple dict and grabbing a key in the middle.""" + + assert json.loads(dict_with_keys) + + stream = adafruit_json_stream.load(BytesChunkIO(dict_with_keys.encode())) + assert stream["field_2"] == 2 + + +def test_simple_dict_with_keys_missing_key_raises(dict_with_keys): + """Test loading a simple dict and grabbing a key that doesn't exist raises.""" + + assert json.loads(dict_with_keys) + + stream = adafruit_json_stream.load(BytesChunkIO(dict_with_keys.encode())) + with pytest.raises(KeyError, match="field_4"): + stream["field_4"] + + +def test_list_with_values(list_with_values): + """Test loading a list and iterating over it.""" + + assert json.loads(list_with_values) + + stream = adafruit_json_stream.load(BytesChunkIO(list_with_values.encode())) + counter = 0 + for value in stream: + counter += 1 + assert value == counter + + +def test_dict_with_list_of_single_entries(dict_with_list_with_single_entries): + """Test loading an dict with a list of dicts with one entry each.""" + + assert json.loads(dict_with_list_with_single_entries) + + stream = adafruit_json_stream.load(BytesChunkIO(dict_with_list_with_single_entries.encode())) + counter = 0 + for obj in stream["list_1"]: + counter += 1 + assert obj["dict_id"] == counter + assert counter == 4 + + +def test_complex_dict(complex_dict): + """Test loading a complex dict.""" + + assert json.loads(complex_dict) + + dict_names = [ + "one", + "two", + "three", + "four", + ] + + stream = adafruit_json_stream.load(BytesChunkIO(complex_dict.encode())) + counter = 0 + sub_counter = 0 + for obj in stream["list_1"]: + counter += 1 + assert obj["dict_id"] == counter + assert obj["dict_name"] == dict_names[counter - 1] + sub_dict = obj["sub_dict"] + assert sub_dict["sub_dict_id"] == counter + 0.1 + assert sub_dict["sub_dict_name"] == f"{dict_names[counter-1]} point one" + for item in obj["sub_list"]: + sub_counter += 1 + assert item == chr(96 + sub_counter) + + assert counter == 2 + assert sub_counter == 6 + + for obj in stream["list_2"]: + counter += 1 + assert obj["dict_id"] == counter + assert obj["dict_name"] == dict_names[counter - 1] + sub_dict = obj["sub_dict"] + assert sub_dict["sub_dict_id"] == counter + 0.1 + assert sub_dict["sub_dict_name"] == f"{dict_names[counter-1]} point one" + for item in obj["sub_list"]: + sub_counter += 1 + assert item == chr(96 + sub_counter) + + assert counter == 4 + assert sub_counter == 12 + + +def test_bad_strings_in_list(list_with_bad_strings): + """Test loading different strings that can confuse the parser.""" + + bad_strings = [ + '"}"', + '{"a": 1, "b": [2,3]}', + '"', + '\\"', + '\\\\"', + '\\x40"', + "[[[{{{", + "]]]}}}", + ] + + assert json.loads(list_with_bad_strings) + + # get each separately + stream = adafruit_json_stream.load(BytesChunkIO(list_with_bad_strings.encode())) + for i, item in enumerate(stream): + assert item == bad_strings[i] + + +def test_bad_strings_in_list_iter(list_with_bad_strings): + """Test loading different strings that can confuse the parser.""" + + bad_strings = [ + '"}"', + '{"a": 1, "b": [2,3]}', + '"', + '\\"', + '\\\\"', + '\\x40"', + "[[[{{{", + "]]]}}}", + ] + + assert json.loads(list_with_bad_strings) + + # get each separately + stream = adafruit_json_stream.load(BytesChunkIO(list_with_bad_strings.encode())) + for i, item in enumerate(stream): + assert item == bad_strings[i] + + +def test_bad_strings_in_dict_as_object(dict_with_bad_strings): + """Test loading different strings that can confuse the parser.""" + + bad_strings = { + "1": '"}"', + "2": '{"a": 1, "b": [2,3]}', + "3": '"', + "4": '\\"', + "5": '\\\\"', + "6": '\\x40"', + "7": "[[[{{{", + "8": "]]]}}}", + } + + # read all at once + stream = adafruit_json_stream.load(BytesChunkIO(dict_with_bad_strings.encode())) + assert stream.as_object() == bad_strings + + +def test_bad_strings_in_dict_all_keys(dict_with_bad_strings): + """Test loading different strings that can confuse the parser.""" + + bad_strings = { + "1": '"}"', + "2": '{"a": 1, "b": [2,3]}', + "3": '"', + "4": '\\"', + "5": '\\\\"', + "6": '\\x40"', + "7": "[[[{{{", + "8": "]]]}}}", + } + + # read one after the other with keys + stream = adafruit_json_stream.load(BytesChunkIO(dict_with_bad_strings.encode())) + assert stream["1"] == bad_strings["1"] + assert stream["2"] == bad_strings["2"] + assert stream["3"] == bad_strings["3"] + assert stream["4"] == bad_strings["4"] + assert stream["5"] == bad_strings["5"] + assert stream["6"] == bad_strings["6"] + assert stream["7"] == bad_strings["7"] + assert stream["8"] == bad_strings["8"] + + +def test_bad_strings_in_dict_skip_some(dict_with_bad_strings): + """Test loading different strings that can confuse the parser.""" + + bad_strings = { + "1": '"}"', + "2": '{"a": 1, "b": [2,3]}', + "3": '"', + "4": '\\"', + "5": '\\\\"', + "6": '\\x40"', + "7": "[[[{{{", + "8": "]]]}}}", + } + + # read some, skip some + stream = adafruit_json_stream.load(BytesChunkIO(dict_with_bad_strings.encode())) + assert stream["2"] == bad_strings["2"] + assert stream["5"] == bad_strings["5"] + assert stream["8"] == bad_strings["8"] + + +def test_complex_dict_grabbing(complex_dict): + """Test loading a complex dict and grabbing specific keys.""" + + assert json.loads(complex_dict) + + stream = adafruit_json_stream.load(BytesChunkIO(complex_dict.encode())) + + list_1 = stream["list_1"] + dict_1 = next(list_1) + sub_list = dict_1["sub_list"] + assert next(sub_list) == "a" + list_2 = stream["list_2"] + next(list_2) + dict_2 = next(list_2) + sub_list = dict_2["sub_list"] + assert next(sub_list) == "j" + + +def test_complex_dict_passed_key_raises(complex_dict): + """ + Test loading a complex dict and attempting to grab a specific key that has been passed raises. + """ + + assert json.loads(complex_dict) + + stream = adafruit_json_stream.load(BytesChunkIO(complex_dict.encode())) + + list_1 = stream["list_1"] + dict_1 = next(list_1) + assert dict_1["dict_name"] == "one" + with pytest.raises(KeyError, match="obects_id"): + stream["obects_id"] + + +def test_complex_dict_passed_reference_raises(complex_dict): + """ + Test loading a complex dict and attempting to grab a data from a saved reference that has + been passed raises. + """ + + assert json.loads(complex_dict) + + stream = adafruit_json_stream.load(BytesChunkIO(complex_dict.encode())) + + list_1 = stream["list_1"] + dict_1 = next(list_1) + sub_dict = dict_1["sub_dict"] + sub_list = dict_1["sub_list"] + list_2 = stream["list_2"] + next(list_2) + with pytest.raises(KeyError, match="sub_dict_id"): + sub_dict["sub_dict_id"] + with pytest.raises(StopIteration): + next(sub_list) + + +# complex_dict is 1518 bytes +@pytest.mark.parametrize( + ("chunk_size", "expected_chunks"), ((10, 152), (50, 31), (100, 16), (5000, 1)) +) +def test_complex_dict_buffer_sizes(chunk_size, complex_dict, expected_chunks): + """Test loading a complex dict and checking the chunking.""" + + assert json.loads(complex_dict) + + bytes_io_chunk = BytesChunkIO(complex_dict.encode(), chunk_size) + + stream = adafruit_json_stream.load(bytes_io_chunk) + + list_1 = stream["list_1"] + dict_1 = next(list_1) + sub_list = dict_1["sub_list"] + assert next(sub_list) == "a" + list_2 = stream["list_2"] + next(list_2) + dict_2 = next(list_2) + sub_list = dict_2["sub_list"] + assert next(sub_list) == "j" + for _ in sub_list: + pass + with pytest.raises(KeyError): + stream["list_3"] + + assert bytes_io_chunk.get_chunks_read() == expected_chunks + assert math.ceil(len(complex_dict) / chunk_size) == expected_chunks + + +# complex_dict is 1518 bytes +@pytest.mark.parametrize(("chunk_size", "expected_chunks"), ((5, 61), (10, 31), (50, 7), (100, 4))) +def test_complex_dict_not_looking_at_all_data_buffer_sizes( + chunk_size, complex_dict, expected_chunks +): + """Test loading a complex dict and checking the chunking.""" + + assert json.loads(complex_dict) + + bytes_io_chunk = BytesChunkIO(complex_dict.encode(), chunk_size) + + stream = adafruit_json_stream.load(bytes_io_chunk) + + list_1 = stream["list_1"] + dict_1 = next(list_1) + sub_list = dict_1["sub_list"] + assert next(sub_list) == "a" + + assert bytes_io_chunk.get_chunks_read() == expected_chunks + assert math.ceil(len(complex_dict) / chunk_size) >= (expected_chunks / 4) + + +def test_incomplete_json_raises(): + """Test incomplete json raises.""" + + data = """ + { + "field_1": 1 + """ + + with pytest.raises(json.JSONDecodeError): + json.loads(data) + + stream = adafruit_json_stream.load(BytesChunkIO(data.encode())) + + with pytest.raises(EOFError): + stream["field_2"] + + +def test_as_object(complex_dict): + """Test loading a complex dict and grabbing parts as objects.""" + + assert json.loads(complex_dict) + + stream = adafruit_json_stream.load(BytesChunkIO(complex_dict.encode())) + + list_1 = stream["list_1"] + dict_1 = next(list_1) + assert dict_1["sub_dict"].as_object() == { + "sub_dict_id": 1.1, + "sub_dict_name": "one point one", + } + assert dict_1["sub_list"].as_object() == ["a", "b", "c"] + dict_2 = next(list_1) + assert dict_2.as_object() == { + "dict_id": 2, + "dict_name": "two", + "sub_dict": {"sub_dict_id": 2.1, "sub_dict_name": "two point one"}, + "sub_list": ["d", "e", "f"], + } + assert stream["list_2"].as_object() == [ + { + "dict_id": 3, + "dict_name": "three", + "sub_dict": {"sub_dict_id": 3.1, "sub_dict_name": "three point one"}, + "sub_list": ["g", "h", "i"], + }, + { + "dict_id": 4, + "dict_name": "four", + "sub_dict": {"sub_dict_id": 4.1, "sub_dict_name": "four point one"}, + "sub_list": ["j", "k", "l"], + }, + ] + + +def test_as_object_stream(dict_with_all_types): + assert json.loads(dict_with_all_types) + + stream = adafruit_json_stream.load(BytesChunkIO(dict_with_all_types.encode())) + + obj = stream.as_object() + assert obj == { + "_check": '{"a": 1, "b": [2,3]}', + "bool": True, + "dict": {"key": "value"}, + "float": 1.1, + "int": 1, + "list": [1, 2, 3], + "null": None, + "string": "string", + } + assert json.loads(obj["_check"]) == { + "a": 1, + "b": [ + 2, + 3, + ], + } + + +def test_as_object_that_is_partially_read_raises(complex_dict): + """Test loading a complex dict and grabbing partially read raises.""" + + assert json.loads(complex_dict) + + stream = adafruit_json_stream.load(BytesChunkIO(complex_dict.encode())) + + list_1 = stream["list_1"] + dict_1 = next(list_1) + assert dict_1["dict_id"] == 1 + with pytest.raises(BufferError): + dict_1.as_object() + + +def test_as_object_grabbing_multiple_subscriptable_levels_twice(complex_dict): + """Test loading a complex dict and grabbing multiple subscriptable levels twice.""" + + assert json.loads(complex_dict) + + stream = adafruit_json_stream.load(BytesChunkIO(complex_dict.encode())) + + list_1 = stream["list_1"] + dict_1 = next(list_1) + assert dict_1["sub_dict"]["sub_dict_id"] == 1.1 + assert dict_1["sub_dict"]["sub_dict_name"] == "one point one" + + +def test_as_object_grabbing_multiple_subscriptable_levels_again_after_passed_raises( + complex_dict, +): + """ + Test loading a complex dict and grabbing multiple subscriptable levels after passing it raises. + """ + + assert json.loads(complex_dict) + + stream = adafruit_json_stream.load(BytesChunkIO(complex_dict.encode())) + + list_1 = stream["list_1"] + dict_1 = next(list_1) + assert dict_1["sub_dict"]["sub_dict_id"] == 1.1 + assert next(dict_1["sub_list"]) == "a" + with pytest.raises(KeyError, match="sub_dict"): + dict_1["sub_dict"]["sub_dict_name"] + + +def test_iterating_keys(dict_with_keys): + """Iterate through keys of a simple object.""" + + bytes_io_chunk = BytesChunkIO(dict_with_keys.encode()) + stream = adafruit_json_stream.load(bytes_io_chunk) + output = list(stream) + assert output == ["field_1", "field_2", "field_3"] + + +def test_iterating_keys_get(dict_with_keys): + """Iterate through keys of a simple object and get values.""" + + the_dict = json.loads(dict_with_keys) + + bytes_io_chunk = BytesChunkIO(dict_with_keys.encode()) + stream = adafruit_json_stream.load(bytes_io_chunk) + for key in stream: + value = stream[key] + assert value == the_dict[key] + + +def test_iterating_items(dict_with_keys): + """Iterate through items of a simple object.""" + + bytes_io_chunk = BytesChunkIO(dict_with_keys.encode()) + stream = adafruit_json_stream.load(bytes_io_chunk) + output = list(stream.items()) + assert output == [("field_1", 1), ("field_2", 2), ("field_3", 3)] + + +def test_iterating_keys_after_get(dict_with_keys): + """Iterate through keys of a simple object after an item has already been read.""" + + bytes_io_chunk = BytesChunkIO(dict_with_keys.encode()) + stream = adafruit_json_stream.load(bytes_io_chunk) + assert stream["field_1"] == 1 + output = list(stream) + assert output == ["field_2", "field_3"] + + +def test_iterating_items_after_get(dict_with_keys): + """Iterate through items of a simple object after an item has already been read.""" + + bytes_io_chunk = BytesChunkIO(dict_with_keys.encode()) + stream = adafruit_json_stream.load(bytes_io_chunk) + assert stream["field_1"] == 1 + output = list(stream.items()) + assert output == [("field_2", 2), ("field_3", 3)] + + +def test_iterating_complex_dict(complex_dict): + """Mix iterating over items of objects in objects in arrays.""" + + names = ["one", "two", "three", "four"] + sub_values = [None, "two point one", "three point one", None] + + stream = adafruit_json_stream.load(BytesChunkIO(complex_dict.encode())) + + thing_num = 0 + for index, item in enumerate(stream.items()): + key, a_list = item + assert key == f"list_{index+1}" + for thing in a_list: + assert thing["dict_name"] == names[thing_num] + for sub_key in thing["sub_dict"]: + # break after getting a key with or without the value + # (testing finish() called from the parent list) + if sub_key == "sub_dict_name": + if thing_num in {1, 2}: + value = thing["sub_dict"][sub_key] + assert value == sub_values[thing_num] + break + thing_num += 1 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..099a9b7 --- /dev/null +++ b/tox.ini @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: 2022 Kevin Conley +# SPDX-FileCopyrightText: 2024 Justin Myers for Adafruit Industries +# +# SPDX-License-Identifier: MIT + +[tox] +envlist = py311 + +[testenv] +description = run tests +deps = + pytest==7.4.3 + requests +commands = pytest + +[testenv:coverage] +description = run coverage +deps = + pytest==7.4.3 + pytest-cov==4.1.0 + requests +package = editable +commands = + coverage run --source=. --omit=tests/* --branch {posargs} -m pytest + coverage report + coverage html + +[testenv:lint] +description = run linters +deps = + pre-commit==3.6.0 +skip_install = true +commands = pre-commit run {posargs} + +[testenv:docs] +description = build docs +deps = + -r requirements.txt + -r docs/requirements.txt +skip_install = true +commands = sphinx-build -E -W -b html docs/. _build/html