#!/usr/bin/env python3

##
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""
Input/Output utilities, including:

 * i/o-specific constants
 * i/o-specific exceptions
 * schema validation
 * leaf value encoding and decoding
 * datum reader/writer stuff (?)

Also includes a generic representation for data, which
uses the following mapping:

  * Schema records are implemented as dict.
  * Schema arrays are implemented as list.
  * Schema maps are implemented as dict.
  * Schema strings are implemented as str.
  * Schema bytes are implemented as bytes.
  * Schema ints are implemented as int.
  * Schema longs are implemented as int.
  * Schema floats are implemented as float.
  * Schema doubles are implemented as float.
  * Schema booleans are implemented as bool.

Validation:

The validation of schema is performed using breadth-first graph
traversal. This allows validation exceptions to pinpoint the exact node
within a complex schema that is problematic, simplifying debugging
considerably. Because it is a traversal, it will also be less
resource-intensive, particularly when validating schema with deep
structures.

Components
==========

Nodes
-----
Avro schemas contain many different schema types. Data about the schema
types is used to validate the data in the corresponding part of a Python
body (the object to be serialized). A node combines a given schema type
with the corresponding Python data, as well as an optional "name" to
identify the specific node. Names are generally the name of a schema
(for named schema) or the name of a field (for child nodes of schema
with named children like maps and records), or None, for schema who's
children are not named (like Arrays).

Iterators
---------
Iterators are generator functions that take a node and return a
generator which will yield a node for each child datum in the data for
the current node. If a node is of a type which has no children, then the
default iterator will immediately exit.

Validators
----------
Validators are used to determine if the datum for a given node is valid
according to the given schema type. Validator functions take a node as
an argument and return a node if the node datum passes validation. If it
does not, the validator must return None.

In most cases, the node returned is identical to the node provided (is
in fact the same object). However, in the case of Union schema, the
returned "valid" node will hold the schema that is represented by the
datum contained. This allows iteration over the child nodes
in that datum, if there are any.
"""

import collections
import datetime
import decimal
import os
import struct
import warnings
from typing import IO, Dict, Generator, Iterable, List, Mapping, Optional, Sequence, Set, Tuple, Union

import avro.constants
import avro.errors
import avro.schema
import avro.timezones

# TODO(hammer): shouldn't ! be < for little-endian (according to spec?)
STRUCT_FLOAT = struct.Struct("<f")  # big-endian float
STRUCT_DOUBLE = struct.Struct("<d")  # big-endian double
STRUCT_SIGNED_SHORT = struct.Struct(">h")  # big-endian signed short
STRUCT_SIGNED_INT = struct.Struct(">i")  # big-endian signed int
STRUCT_SIGNED_LONG = struct.Struct(">q")  # big-endian signed long

ValidationNode = collections.namedtuple("ValidationNode", ["schema", "datum", "name"])
ValidationNodeGeneratorType = Generator[ValidationNode, None, None]
JsonScalarFieldType = Union[None, bool, str, int, float]


def validate(expected_schema: avro.schema.Schema, datum: object, raise_on_error: bool = False) -> bool:
    """Return True if the provided datum is valid for the expected schema

    If raise_on_error is passed and True, then raise a validation error
    with specific information about the error encountered in validation.

    :param expected_schema: An avro schema type object representing the schema against
                            which the datum will be validated.
    :param datum: The datum to be validated, A python dictionary or some supported type
    :param raise_on_error: True if a AvroTypeException should be raised immediately when a
                           validation problem is encountered.
    :raises: AvroTypeException if datum is invalid and raise_on_error is True
    :returns: True if datum is valid for expected_schema, False if not.
    """
    # use a FIFO queue to process schema nodes breadth first.
    nodes = collections.deque([ValidationNode(expected_schema, datum, getattr(expected_schema, "name", None))])

    while nodes:
        current_node = nodes.popleft()

        # _validate_node returns the node for iteration if it is valid. Or it returns None
        validated_schema = current_node.schema.validate(current_node.datum)
        valid_node = ValidationNode(validated_schema, current_node.datum, current_node.name) if validated_schema else None

        if valid_node is None:
            if raise_on_error:
                raise avro.errors.AvroTypeException(current_node.schema, current_node.name, current_node.datum)
            return False  # preserve the prior validation behavior of returning false when there are problems.
        # if there are children of this node to append, do so.
        for child_node in _iterate_node(valid_node):
            nodes.append(child_node)

    return True


def _iterate_node(node: ValidationNode) -> ValidationNodeGeneratorType:
    for item in _ITERATORS.get(node.schema.type, _default_iterator)(node):
        yield ValidationNode(*item)


#############
# Iteration #
#############


def _default_iterator(_) -> ValidationNodeGeneratorType:
    """Immediately raise StopIteration.

    This exists to prevent problems with iteration over unsupported container types.
    """
    yield from ()


def _record_iterator(node: ValidationNode) -> ValidationNodeGeneratorType:
    """Yield each child node of the provided record node."""
    schema, datum, _ = node
    return (ValidationNode(field.type, datum.get(field.name), field.name) for field in schema.fields)


def _array_iterator(node: ValidationNode) -> ValidationNodeGeneratorType:
    """Yield each child node of the provided array node."""
    schema, datum, name = node
    return (ValidationNode(schema.items, item, name) for item in datum)


def _map_iterator(node: ValidationNode) -> ValidationNodeGeneratorType:
    """Yield each child node of the provided map node."""
    schema, datum, _ = node
    child_schema = schema.values
    return (ValidationNode(child_schema, child_datum, child_name) for child_name, child_datum in datum.items())


_ITERATORS = {
    "record": _record_iterator,
    "array": _array_iterator,
    "map": _map_iterator,
}
_ITERATORS["error"] = _ITERATORS["request"] = _ITERATORS["record"]


#
# Decoder/Encoder
#


class BinaryDecoder:
    """Read leaf values."""

    _reader: IO[bytes]

    # Reads with a declared length above this many bytes are validated against
    # the number of bytes actually remaining (when the reader is seekable)
    # before allocating, to guard against an out-of-memory attack from a
    # malicious or truncated input. Smaller reads skip the check to avoid
    # per-value overhead, since they cannot cause a meaningful over-allocation.
    _MAX_UNCHECKED_READ = 1024 * 1024

    def __init__(self, reader: IO[bytes]) -> None:
        """
        reader is a Python object on which we can call read, seek, and tell.
        """
        self._reader = reader

    @property
    def reader(self) -> IO[bytes]:
        return self._reader

    def read(self, n: int) -> bytes:
        """
        Read n bytes.
        """
        if n < 0:
            raise avro.errors.InvalidAvroBinaryEncoding(f"Requested {n} bytes to read, expected positive integer.")
        if n > self._MAX_UNCHECKED_READ:
            remaining = self.bytes_remaining()
            if remaining is not None and n > remaining:
                raise avro.errors.InvalidAvroBinaryEncoding(f"Requested {n} bytes to read, but only {remaining} remain.")
        read_bytes = self.reader.read(n)
        if len(read_bytes) != n:
            raise avro.errors.InvalidAvroBinaryEncoding(f"Read {len(read_bytes)} bytes, expected {n} bytes")
        return read_bytes

    def bytes_remaining(self) -> Optional[int]:
        """
        Return the number of bytes still available to read, or ``None`` when
        that count is not known (a non-seekable reader, or one whose position
        cannot be obtained). Used to reject a declared length or collection
        block count that exceeds the data actually available before allocating
        for it.
        """
        reader = self.reader
        try:
            pos = reader.tell()
        except (OSError, ValueError, AttributeError, TypeError):
            # Not seekable, or the position could not be determined.
            return None
        try:
            reader.seek(0, os.SEEK_END)
            end = reader.tell()
            # Clamp to 0: a stream positioned past its end (seekable file-like
            # objects allow seeking beyond EOF) would otherwise report a negative
            # "remaining", violating the non-negative contract.
            return max(0, end - pos)
        except (OSError, ValueError, AttributeError, TypeError):
            return None
        finally:
            # Always restore the original position, even if seeking to the end
            # or reading it failed, so the reader is never left at EOF.
            try:
                reader.seek(pos)
            except (OSError, ValueError, AttributeError, TypeError):
                pass

    def read_null(self) -> None:
        """
        null is written as zero bytes
        """
        return None

    def read_boolean(self) -> bool:
        """
        a boolean is written as a single byte
        whose value is either 0 (false) or 1 (true).
        """
        return ord(self.read(1)) == 1

    def read_int(self) -> int:
        """
        int and long values are written using variable-length, zig-zag coding.
        """
        return self.read_long()

    def read_long(self) -> int:
        """
        int and long values are written using variable-length, zig-zag coding.
        """
        b = ord(self.read(1))
        n = b & 0x7F
        shift = 7
        while (b & 0x80) != 0:
            # A 64-bit value needs at most 10 bytes (shifts 0..63); reject an
            # overlong varint rather than accepting a malformed, arbitrarily
            # large value.
            if shift >= 70:
                raise avro.errors.InvalidAvroBinaryEncoding("Varint is too long")
            b = ord(self.read(1))
            # The 10th byte (shift == 63) contributes only bit 63; any higher
            # payload bit would push the value outside the 64-bit range, so a
            # valid zig-zag long must have them clear.
            if shift == 63 and (b & 0x7E) != 0:
                raise avro.errors.InvalidAvroBinaryEncoding("Varint is too long")
            n |= (b & 0x7F) << shift
            shift += 7
        datum = (n >> 1) ^ -(n & 1)
        return datum

    def read_float(self) -> float:
        """
        A float is written as 4 bytes.
        The float is converted into a 32-bit integer using a method equivalent to
        Java's floatToRawIntBits and then encoded in little-endian format.
        """
        return float(STRUCT_FLOAT.unpack(self.read(4))[0])

    def read_double(self) -> float:
        """
        A double is written as 8 bytes.
        The double is converted into a 64-bit integer using a method equivalent to
        Java's doubleToRawLongBits and then encoded in little-endian format.
        """
        return float(STRUCT_DOUBLE.unpack(self.read(8))[0])

    def read_decimal_from_bytes(self, precision: int, scale: int) -> decimal.Decimal:
        """
        Decimal bytes are decoded as signed short, int or long depending on the
        size of bytes.
        """
        size = self.read_long()
        return self.read_decimal_from_fixed(precision, scale, size)

    def read_decimal_from_fixed(self, precision: int, scale: int, size: int) -> decimal.Decimal:
        """
        Decimal is encoded as fixed. Fixed instances are encoded using the
        number of bytes declared in the schema.
        """
        datum = self.read(size)
        unscaled_datum = 0
        msb = struct.unpack("!b", datum[0:1])[0]
        leftmost_bit = (msb >> 7) & 1
        if leftmost_bit == 1:
            modified_first_byte = ord(datum[0:1]) ^ (1 << 7)
            datum = bytearray([modified_first_byte]) + datum[1:]
            for offset in range(size):
                unscaled_datum <<= 8
                unscaled_datum += ord(datum[offset : 1 + offset])
            unscaled_datum += pow(-2, (size * 8) - 1)
        else:
            for offset in range(size):
                unscaled_datum <<= 8
                unscaled_datum += ord(datum[offset : 1 + offset])

        original_prec = decimal.getcontext().prec
        try:
            decimal.getcontext().prec = precision
            scaled_datum = decimal.Decimal(unscaled_datum).scaleb(-scale)
        finally:
            decimal.getcontext().prec = original_prec
        return scaled_datum

    def read_bytes(self) -> bytes:
        """
        Bytes are encoded as a long followed by that many bytes of data.
        """
        return self.read(self.read_long())

    def read_utf8(self) -> str:
        """
        A string is encoded as a long followed by
        that many bytes of UTF-8 encoded character data.
        """
        return self.read_bytes().decode("utf-8")

    def read_date_from_int(self) -> datetime.date:
        """
        int is decoded as python date object.
        int stores the number of days from
        the unix epoch, 1 January 1970 (ISO calendar).
        """
        days_since_epoch = self.read_int()
        return datetime.date(1970, 1, 1) + datetime.timedelta(days_since_epoch)

    def _build_time_object(self, value: int, scale_to_micro: int) -> datetime.time:
        value = value * scale_to_micro
        value, microseconds = divmod(value, 1000000)
        value, seconds = divmod(value, 60)
        value, minutes = divmod(value, 60)
        hours = value

        return datetime.time(hour=hours, minute=minutes, second=seconds, microsecond=microseconds)

    def read_time_millis_from_int(self) -> datetime.time:
        """
        int is decoded as python time object which represents
        the number of milliseconds after midnight, 00:00:00.000.
        """
        milliseconds = self.read_int()
        return self._build_time_object(milliseconds, 1000)

    def read_time_micros_from_long(self) -> datetime.time:
        """
        long is decoded as python time object which represents
        the number of microseconds after midnight, 00:00:00.000000.
        """
        microseconds = self.read_long()
        return self._build_time_object(microseconds, 1)

    def read_timestamp_millis_from_long(self) -> datetime.datetime:
        """
        long is decoded as python datetime object which represents
        the number of milliseconds from the unix epoch, 1 January 1970.
        """
        timestamp_millis = self.read_long()
        timedelta = datetime.timedelta(microseconds=timestamp_millis * 1000)
        unix_epoch_datetime = datetime.datetime(1970, 1, 1, 0, 0, 0, 0, tzinfo=avro.timezones.utc)
        return unix_epoch_datetime + timedelta

    def read_timestamp_micros_from_long(self) -> datetime.datetime:
        """
        long is decoded as python datetime object which represents
        the number of microseconds from the unix epoch, 1 January 1970.
        """
        timestamp_micros = self.read_long()
        timedelta = datetime.timedelta(microseconds=timestamp_micros)
        unix_epoch_datetime = datetime.datetime(1970, 1, 1, 0, 0, 0, 0, tzinfo=avro.timezones.utc)
        return unix_epoch_datetime + timedelta

    def skip_null(self) -> None:
        pass

    def skip_boolean(self) -> None:
        self.skip(1)

    def skip_int(self) -> None:
        self.skip_long()

    def skip_long(self) -> None:
        b = ord(self.read(1))
        shift = 7
        while (b & 0x80) != 0:
            # A 64-bit varint is at most 10 bytes; reject an overlong chain so a
            # skipped long can't force scanning unbounded input (read_long caps
            # the same way).
            if shift >= 70:
                raise avro.errors.InvalidAvroBinaryEncoding("Varint is too long")
            b = ord(self.read(1))
            # The 10th byte (shift == 63) contributes only bit 63; any higher
            # payload bit would push the value outside the 64-bit range. Reject
            # it here too so skip_long enforces the same validity as read_long
            # (e.g. a malformed negative-block byte-size in read_array/read_map).
            if shift == 63 and (b & 0x7E) != 0:
                raise avro.errors.InvalidAvroBinaryEncoding("Varint is too long")
            shift += 7

    def skip_float(self) -> None:
        self.skip(4)

    def skip_double(self) -> None:
        self.skip(8)

    def skip_bytes(self) -> None:
        # The length prefix is attacker-controlled: an oversized value would seek
        # past EOF (a negative one is rejected by ``skip`` below, which is the
        # single place that guards backward seeks). Validate an oversized length
        # against the bytes remaining the same way ``read`` does before skipping.
        n = self.read_long()
        if n > self._MAX_UNCHECKED_READ:
            remaining = self.bytes_remaining()
            if remaining is not None and n > remaining:
                raise avro.errors.InvalidAvroBinaryEncoding(f"Requested {n} bytes to skip, but only {remaining} remain.")
        self.skip(n)

    def skip_utf8(self) -> None:
        self.skip_bytes()

    def skip(self, n: int) -> None:
        # Guard against a negative skip (a backward seek), which would corrupt
        # the decoder position. Callers pass schema-derived sizes (fixed/float/
        # double) or already-validated lengths, so this is a defensive backstop.
        if n < 0:
            raise avro.errors.InvalidAvroBinaryEncoding(f"Cannot skip a negative number of bytes: {n}")
        self.reader.seek(self.reader.tell() + n)


class BinaryEncoder:
    """Write leaf values."""

    _writer: IO[bytes]

    def __init__(self, writer: IO[bytes]) -> None:
        """
        writer is a Python object on which we can call write.
        """
        self._writer = writer

    @property
    def writer(self) -> IO[bytes]:
        return self._writer

    def write(self, datum: bytes) -> None:
        """Write an arbitrary datum."""
        self.writer.write(datum)

    def write_null(self, datum: None) -> None:
        """
        null is written as zero bytes
        """

    def write_boolean(self, datum: bool) -> None:
        """
        a boolean is written as a single byte
        whose value is either 0 (false) or 1 (true).
        """
        self.write(bytearray([bool(datum)]))

    def write_int(self, datum: int) -> None:
        """
        int and long values are written using variable-length, zig-zag coding.
        """
        self.write_long(datum)

    def write_long(self, datum: int) -> None:
        """
        int and long values are written using variable-length, zig-zag coding.
        """
        datum = (datum << 1) ^ (datum >> 63)
        while (datum & ~0x7F) != 0:
            self.write(bytearray([(datum & 0x7F) | 0x80]))
            datum >>= 7
        self.write(bytearray([datum]))

    def write_float(self, datum: float) -> None:
        """
        A float is written as 4 bytes.
        The float is converted into a 32-bit integer using a method equivalent to
        Java's floatToRawIntBits and then encoded in little-endian format.
        """
        self.write(STRUCT_FLOAT.pack(datum))

    def write_double(self, datum: float) -> None:
        """
        A double is written as 8 bytes.
        The double is converted into a 64-bit integer using a method equivalent to
        Java's doubleToRawLongBits and then encoded in little-endian format.
        """
        self.write(STRUCT_DOUBLE.pack(datum))

    def write_decimal_bytes(self, datum: decimal.Decimal, scale: int) -> None:
        """
        Decimal in bytes are encoded as long. Since size of packed value in bytes for
        signed long is 8, 8 bytes are written.
        """
        sign, digits, exp = datum.as_tuple()
        if (-1 * int(exp)) > scale:
            raise avro.errors.AvroOutOfScaleException(scale, datum, exp)

        unscaled_datum = 0
        for digit in digits:
            unscaled_datum = (unscaled_datum * 10) + digit

        bits_req = unscaled_datum.bit_length() + 1
        if sign:
            unscaled_datum = (1 << bits_req) - unscaled_datum

        bytes_req = bits_req // 8
        padding_bits = ~((1 << bits_req) - 1) if sign else 0
        packed_bits = padding_bits | unscaled_datum

        bytes_req += 1 if (bytes_req << 3) < bits_req else 0
        self.write_long(bytes_req)
        for index in range(bytes_req - 1, -1, -1):
            bits_to_write = packed_bits >> (8 * index)
            self.write(bytearray([bits_to_write & 0xFF]))

    def write_decimal_fixed(self, datum: decimal.Decimal, scale: int, size: int) -> None:
        """
        Decimal in fixed are encoded as size of fixed bytes.
        """
        sign, digits, exp = datum.as_tuple()
        if (-1 * int(exp)) > scale:
            raise avro.errors.AvroOutOfScaleException(scale, datum, exp)

        unscaled_datum = 0
        for digit in digits:
            unscaled_datum = (unscaled_datum * 10) + digit

        bits_req = unscaled_datum.bit_length() + 1
        size_in_bits = size * 8
        offset_bits = size_in_bits - bits_req

        mask = 2**size_in_bits - 1
        bit = 1
        for i in range(bits_req):
            mask ^= bit
            bit <<= 1

        if bits_req < 8:
            bytes_req = 1
        else:
            bytes_req = bits_req // 8
            if bits_req % 8 != 0:
                bytes_req += 1
        if sign:
            unscaled_datum = (1 << bits_req) - unscaled_datum
            unscaled_datum = mask | unscaled_datum
            for index in range(size - 1, -1, -1):
                bits_to_write = unscaled_datum >> (8 * index)
                self.write(bytearray([bits_to_write & 0xFF]))
        else:
            for i in range(offset_bits // 8):
                self.write(b"\x00")
            for index in range(bytes_req - 1, -1, -1):
                bits_to_write = unscaled_datum >> (8 * index)
                self.write(bytearray([bits_to_write & 0xFF]))

    def write_bytes(self, datum: bytes) -> None:
        """
        Bytes are encoded as a long followed by that many bytes of data.
        """
        self.write_long(len(datum))
        self.write(struct.pack(f"{len(datum)}s", datum))

    def write_utf8(self, datum: str) -> None:
        """
        A string is encoded as a long followed by
        that many bytes of UTF-8 encoded character data.
        """
        self.write_bytes(datum.encode("utf-8"))

    def write_date_int(self, datum: datetime.date) -> None:
        """
        Encode python date object as int.
        It stores the number of days from
        the unix epoch, 1 January 1970 (ISO calendar).
        """
        delta_date = datum - datetime.date(1970, 1, 1)
        self.write_int(delta_date.days)

    def write_time_millis_int(self, datum: datetime.time) -> None:
        """
        Encode python time object as int.
        It stores the number of milliseconds from midnight, 00:00:00.000
        """
        milliseconds = datum.hour * 3600000 + datum.minute * 60000 + datum.second * 1000 + datum.microsecond // 1000
        self.write_int(milliseconds)

    def write_time_micros_long(self, datum: datetime.time) -> None:
        """
        Encode python time object as long.
        It stores the number of microseconds from midnight, 00:00:00.000000
        """
        microseconds = datum.hour * 3600000000 + datum.minute * 60000000 + datum.second * 1000000 + datum.microsecond
        self.write_long(microseconds)

    def _timedelta_total_microseconds(self, timedelta_: datetime.timedelta) -> int:
        return timedelta_.microseconds + (timedelta_.seconds + timedelta_.days * 24 * 3600) * 10**6

    def write_timestamp_millis_long(self, datum: datetime.datetime) -> None:
        """
        Encode python datetime object as long.
        It stores the number of milliseconds from midnight of unix epoch, 1 January 1970.
        """
        datum = datum.astimezone(tz=avro.timezones.utc)
        timedelta = datum - datetime.datetime(1970, 1, 1, 0, 0, 0, 0, tzinfo=avro.timezones.utc)
        milliseconds = self._timedelta_total_microseconds(timedelta) // 1000
        self.write_long(milliseconds)

    def write_timestamp_micros_long(self, datum: datetime.datetime) -> None:
        """
        Encode python datetime object as long.
        It stores the number of microseconds from midnight of unix epoch, 1 January 1970.
        """
        datum = datum.astimezone(tz=avro.timezones.utc)
        timedelta = datum - datetime.datetime(1970, 1, 1, 0, 0, 0, 0, tzinfo=avro.timezones.utc)
        microseconds = self._timedelta_total_microseconds(timedelta)
        self.write_long(microseconds)


#
# DatumReader/Writer
#

# Environment variable overriding the collection element limits. When set to a
# non-negative integer it caps both the number of zero-byte-encoded collection
# elements (e.g. an array of nulls) and the structural cap on the total number of
# elements in any collection, allocated from a single decode.
MAX_COLLECTION_ITEMS_ENV = "AVRO_MAX_COLLECTION_ITEMS"

# Default maximum number of zero-byte-encoded collection elements to allocate
# across a single decoded datum. Elements whose schema encodes to zero bytes
# (``null``, a zero-length ``fixed``, or a record with only zero-byte fields)
# consume no input, so the bytes-remaining check cannot bound their count; without
# a cap a tiny payload can declare a huge block count and exhaust memory. The cap
# is cumulative over the whole datum rather than per collection, because a record's
# schema can declare many such collection fields, each individually under the limit
# but jointly unbounded. A legitimate datum of zero-byte elements is small, so this
# default is generous while still rejecting pathological input. It can be raised (or
# lowered) with the ``AVRO_MAX_COLLECTION_ITEMS`` environment variable.
DEFAULT_MAX_COLLECTION_ITEMS = 10_000_000

# Default structural cap on the number of elements in any array or map (a
# collection larger than this is treated as malformed regardless of element
# type). Matches the historical ``Integer.MAX_VALUE - 8`` limit. Elements with a
# positive on-wire size are also bounded by the bytes remaining; this cap is an
# additional overflow/defense-in-depth guard. ``AVRO_MAX_COLLECTION_ITEMS``, when
# set, overrides both this and ``DEFAULT_MAX_COLLECTION_ITEMS``.
DEFAULT_MAX_COLLECTION_STRUCTURAL = (1 << 31) - 1 - 8  # Integer.MAX_VALUE - 8

# Block counts at or below this are not checked against the bytes remaining. A
# collection element with a positive on-wire size must be backed by real bytes,
# so a small block cannot over-allocate meaningfully; skipping the check avoids a
# per-block ``bytes_remaining()`` seek (tell + seek-to-end + seek-back), which is
# not free on a real file object. Mirrors ``BinaryDecoder._MAX_UNCHECKED_READ``.
# The structural cap below is always enforced (no seek), and zero-byte elements
# keep their own cumulative cap.
_MAX_UNCHECKED_COLLECTION = 1024


def _collection_limits() -> Tuple[int, int]:
    """Return ``(zero_byte_limit, structural_limit)``.

    ``AVRO_MAX_COLLECTION_ITEMS``, when set to a non-negative integer, pins *both*
    limits to that single value. The coupling runs in both directions: raising it
    to lift the zero-byte-element limit also lowers the structural cap from
    ``DEFAULT_MAX_COLLECTION_STRUCTURAL`` (~2.1 billion) to that same value, and
    lowering it tightens both. Set it above ``DEFAULT_MAX_COLLECTION_STRUCTURAL``
    if you need to raise the zero-byte limit without reducing the structural cap.
    When unset, zero-byte elements use the tighter ``DEFAULT_MAX_COLLECTION_ITEMS``
    and all collections use ``DEFAULT_MAX_COLLECTION_STRUCTURAL``.
    """
    value = os.environ.get(MAX_COLLECTION_ITEMS_ENV)
    if value is None:
        return DEFAULT_MAX_COLLECTION_ITEMS, DEFAULT_MAX_COLLECTION_STRUCTURAL
    try:
        parsed = int(value)
    except ValueError:
        warnings.warn(avro.errors.AvroWarning(f"Ignoring invalid {MAX_COLLECTION_ITEMS_ENV} value: {value!r}"))
        return DEFAULT_MAX_COLLECTION_ITEMS, DEFAULT_MAX_COLLECTION_STRUCTURAL
    if parsed < 0:
        warnings.warn(avro.errors.AvroWarning(f"Ignoring negative {MAX_COLLECTION_ITEMS_ENV} value: {value!r}"))
        return DEFAULT_MAX_COLLECTION_ITEMS, DEFAULT_MAX_COLLECTION_STRUCTURAL
    return parsed, parsed


def _max_collection_items() -> int:
    """Return the configured zero-byte-element collection limit."""
    return _collection_limits()[0]


def _min_bytes_per_element(schema: avro.schema.Schema, visited: Optional[Set[int]] = None) -> int:
    """
    Return the minimum number of bytes a single value of ``schema`` can occupy
    on the wire. Used to reject an array/map block count that could not possibly
    be backed by the bytes remaining. A type that can encode to zero bytes
    (``null``) returns 0, which disables the collection check for it (avoiding a
    false positive on, e.g., an array of nulls).
    """
    if visited is None:
        visited = set()
    schema_type = schema.type
    if schema_type == "null":
        return 0
    if schema_type == "float":
        return 4
    if schema_type == "double":
        return 8
    if schema_type == "fixed":
        return getattr(schema, "size", 1)
    if schema_type in ("record", "error"):
        # Guard against self-referencing records (recursion would not terminate).
        # A recursive reference is not a zero-byte value: any finite recursive
        # value must terminate through a union (>= 1 byte branch index) or an
        # empty array/map (>= 1 byte block count), so 1 is a safe conservative
        # lower bound. Returning 0 here would wrongly treat recursive records as
        # zero-byte elements and weaken the bytes-remaining precheck.
        if id(schema) in visited:
            return 1
        visited.add(id(schema))
        total = 0
        for field in getattr(schema, "fields", []):
            total += _min_bytes_per_element(field.type, visited)
        visited.discard(id(schema))
        return total
    if schema_type == "union":
        # A union encodes a >= 1 byte branch index plus the selected branch's
        # payload, so its minimum is 1 + the smallest branch minimum (which is
        # 1 when any branch is null / zero-byte).
        branches = getattr(schema, "schemas", [])
        if not branches:
            return 1
        return 1 + min(_min_bytes_per_element(branch, visited) for branch in branches)
    # boolean, int, long, bytes, string, enum, array, map: all >= 1 byte.
    return 1


class DatumReader:
    """Deserialize Avro-encoded data into a Python data structure."""

    _writers_schema: Optional[avro.schema.Schema]
    _readers_schema: Optional[avro.schema.Schema]

    def __init__(self, writers_schema: Optional[avro.schema.Schema] = None, readers_schema: Optional[avro.schema.Schema] = None) -> None:
        """
        As defined in the Avro specification, we call the schema encoded
        in the data the "writer's schema", and the schema expected by the
        reader the "reader's schema".
        """
        self._writers_schema = writers_schema
        self._readers_schema = readers_schema
        # Cumulative number of zero-byte-encoded collection elements (e.g. an
        # array of nulls) allocated while decoding the *current* datum. Reset at
        # the start of each top-level read(). Because such elements consume no
        # input bytes, the bytes-remaining check cannot bound them; capping the
        # count per collection is not enough either, since a single record's
        # schema can declare many collection fields, each individually under the
        # limit but together unbounded. The cap is therefore applied across the
        # whole datum. See _ensure_collection_available.
        self._zero_byte_items_read = 0

    @property
    def writers_schema(self) -> Optional[avro.schema.Schema]:
        return self._writers_schema

    @writers_schema.setter
    def writers_schema(self, writers_schema: avro.schema.Schema) -> None:
        self._writers_schema = writers_schema

    @property
    def readers_schema(self) -> Optional[avro.schema.Schema]:
        return self._readers_schema

    @readers_schema.setter
    def readers_schema(self, readers_schema: avro.schema.Schema) -> None:
        self._readers_schema = readers_schema

    def read(self, decoder: "BinaryDecoder") -> object:
        if self.writers_schema is None:
            raise avro.errors.IONotReadyException("Cannot read without a writer's schema.")
        reader_schema = self.readers_schema
        if reader_schema is None:
            reader_schema = self.writers_schema
        # Start a fresh zero-byte-element budget for this datum. The cap bounds
        # the *cumulative* number of zero-byte collection elements decoded across
        # every collection in this datum, not per collection, so a record made of
        # many small collection fields cannot bypass it (see
        # _ensure_collection_available).
        self._zero_byte_items_read = 0
        return self.read_data(self.writers_schema, reader_schema, decoder)

    def read_data(self, writers_schema: avro.schema.Schema, readers_schema: avro.schema.Schema, decoder: "BinaryDecoder") -> object:
        # schema matching
        if not readers_schema.match(writers_schema):
            raise avro.errors.SchemaResolutionException("Schemas do not match.", writers_schema, readers_schema)

        logical_type = getattr(writers_schema, "logical_type", None)

        # function dispatch for reading data based on type of writer's schema
        if isinstance(writers_schema, avro.schema.UnionSchema) and isinstance(readers_schema, avro.schema.UnionSchema):
            return self.read_union(writers_schema, readers_schema, decoder)

        if isinstance(readers_schema, avro.schema.UnionSchema):
            # schema resolution: reader's schema is a union, writer's schema is not
            for s in readers_schema.schemas:
                if s.match(writers_schema):
                    return self.read_data(writers_schema, s, decoder)

            # This shouldn't happen because of the match check at the start of this method.
            raise avro.errors.SchemaResolutionException("Schemas do not match.", writers_schema, readers_schema)

        if writers_schema.type == "null":
            return None
        if writers_schema.type == "boolean":
            return decoder.read_boolean()
        if writers_schema.type == "string":
            return decoder.read_utf8()
        if writers_schema.type == "int":
            if logical_type == avro.constants.DATE:
                return decoder.read_date_from_int()
            if logical_type == avro.constants.TIME_MILLIS:
                return decoder.read_time_millis_from_int()
            return decoder.read_int()
        if writers_schema.type == "long":
            if logical_type == avro.constants.TIME_MICROS:
                return decoder.read_time_micros_from_long()
            if logical_type == avro.constants.TIMESTAMP_MILLIS:
                return decoder.read_timestamp_millis_from_long()
            if logical_type == avro.constants.TIMESTAMP_MICROS:
                return decoder.read_timestamp_micros_from_long()
            return decoder.read_long()
        if writers_schema.type == "float":
            return decoder.read_float()
        if writers_schema.type == "double":
            return decoder.read_double()
        if writers_schema.type == "bytes":
            if logical_type == "decimal":
                precision = writers_schema.get_prop("precision")
                if not (isinstance(precision, int) and precision > 0):
                    warnings.warn(avro.errors.IgnoredLogicalType(f"Invalid decimal precision {precision}. Must be a positive integer."))
                    return decoder.read_bytes()
                scale = writers_schema.get_prop("scale")
                if not (isinstance(scale, int) and scale >= 0):
                    warnings.warn(avro.errors.IgnoredLogicalType(f"Invalid decimal scale {scale}. Must be a non-negative integer."))
                    return decoder.read_bytes()
                return decoder.read_decimal_from_bytes(precision, scale)
            return decoder.read_bytes()
        if isinstance(writers_schema, avro.schema.FixedSchema) and isinstance(readers_schema, avro.schema.FixedSchema):
            if logical_type == "decimal":
                precision = writers_schema.get_prop("precision")
                if not (isinstance(precision, int) and precision > 0):
                    warnings.warn(avro.errors.IgnoredLogicalType(f"Invalid decimal precision {precision}. Must be a positive integer."))
                    return self.read_fixed(writers_schema, readers_schema, decoder)
                scale = writers_schema.get_prop("scale")
                if not (isinstance(scale, int) and scale >= 0):
                    warnings.warn(avro.errors.IgnoredLogicalType(f"Invalid decimal scale {scale}. Must be a non-negative integer."))
                    return self.read_fixed(writers_schema, readers_schema, decoder)
                return decoder.read_decimal_from_fixed(precision, scale, writers_schema.size)
            return self.read_fixed(writers_schema, readers_schema, decoder)
        if isinstance(writers_schema, avro.schema.EnumSchema) and isinstance(readers_schema, avro.schema.EnumSchema):
            return self.read_enum(writers_schema, readers_schema, decoder)
        if isinstance(writers_schema, avro.schema.ArraySchema) and isinstance(readers_schema, avro.schema.ArraySchema):
            return self.read_array(writers_schema, readers_schema, decoder)
        if isinstance(writers_schema, avro.schema.MapSchema) and isinstance(readers_schema, avro.schema.MapSchema):
            return self.read_map(writers_schema, readers_schema, decoder)
        if isinstance(writers_schema, avro.schema.RecordSchema) and isinstance(readers_schema, avro.schema.RecordSchema):
            # .type in ["record", "error", "request"]:
            return self.read_record(writers_schema, readers_schema, decoder)
        raise avro.errors.AvroException(f"Cannot read unknown schema type: {writers_schema.type}")

    def skip_data(self, writers_schema: avro.schema.Schema, decoder: BinaryDecoder) -> None:
        if writers_schema.type == "null":
            return decoder.skip_null()
        if writers_schema.type == "boolean":
            return decoder.skip_boolean()
        if writers_schema.type == "string":
            return decoder.skip_utf8()
        if writers_schema.type == "int":
            return decoder.skip_int()
        if writers_schema.type == "long":
            return decoder.skip_long()
        if writers_schema.type == "float":
            return decoder.skip_float()
        if writers_schema.type == "double":
            return decoder.skip_double()
        if writers_schema.type == "bytes":
            return decoder.skip_bytes()
        if isinstance(writers_schema, avro.schema.FixedSchema):
            return self.skip_fixed(writers_schema, decoder)
        if isinstance(writers_schema, avro.schema.EnumSchema):
            return self.skip_enum(writers_schema, decoder)
        if isinstance(writers_schema, avro.schema.ArraySchema):
            return self.skip_array(writers_schema, decoder)
        if isinstance(writers_schema, avro.schema.MapSchema):
            return self.skip_map(writers_schema, decoder)
        if isinstance(writers_schema, avro.schema.UnionSchema):
            return self.skip_union(writers_schema, decoder)
        if isinstance(writers_schema, avro.schema.RecordSchema):
            return self.skip_record(writers_schema, decoder)
        raise avro.errors.AvroException(f"Unknown schema type: {writers_schema.type}")

    def read_fixed(self, writers_schema: avro.schema.FixedSchema, readers_schema: avro.schema.Schema, decoder: BinaryDecoder) -> bytes:
        """
        Fixed instances are encoded using the number of bytes declared
        in the schema.
        """
        return decoder.read(writers_schema.size)

    def skip_fixed(self, writers_schema: avro.schema.FixedSchema, decoder: BinaryDecoder) -> None:
        return decoder.skip(writers_schema.size)

    def read_enum(self, writers_schema: avro.schema.EnumSchema, readers_schema: avro.schema.EnumSchema, decoder: BinaryDecoder) -> str:
        """
        An enum is encoded by a int, representing the zero-based position
        of the symbol in the schema.
        """
        # read data
        index_of_symbol = decoder.read_int()
        if index_of_symbol < 0:
            # A negative index is malformed data and never resolves to a default.
            raise avro.errors.SchemaResolutionException(
                f"Can't access enum index {index_of_symbol} for enum with {len(writers_schema.symbols)} symbols", writers_schema, readers_schema
            )
        if index_of_symbol >= len(writers_schema.symbols):
            # A symbol beyond the writer's schema may be a future symbol; fall
            # back to the schema's default when one is defined.
            default = writers_schema.default
            if default is not None:
                return default
            raise avro.errors.SchemaResolutionException(
                f"Can't access enum index {index_of_symbol} for enum with {len(writers_schema.symbols)} symbols", writers_schema, readers_schema
            )
        read_symbol = writers_schema.symbols[index_of_symbol]

        # schema resolution
        if read_symbol not in readers_schema.symbols:
            raise avro.errors.SchemaResolutionException(f"Symbol {read_symbol} not present in Reader's Schema", writers_schema, readers_schema)

        return read_symbol

    def skip_enum(self, writers_schema: avro.schema.EnumSchema, decoder: BinaryDecoder) -> None:
        return decoder.skip_int()

    def _ensure_collection_available(
        self,
        decoder: BinaryDecoder,
        existing: int,
        count: int,
        min_bytes_per_element: int,
        zero_byte_limit: int,
        structural_limit: int,
    ) -> None:
        """
        Reject a collection (array or map) block that could not be backed by the
        input, before iterating.

        For elements with a positive minimum on-wire size, the declared count is
        checked against the bytes actually remaining and against a structural
        limit (an overflow/defense-in-depth cap); both are naturally bounded per
        collection because decoding consumes input, so ``existing`` is the count
        already read in *this* collection.

        For zero-byte elements (e.g. an array of nulls), which consume no input,
        neither the bytes remaining nor a per-collection count can bound them: a
        single datum's schema may declare many such collections (one per record
        field), each individually small but jointly unbounded. Their count is
        therefore accumulated on the reader across the whole datum
        (``self._zero_byte_items_read``, reset per top-level ``read()``) and
        checked against the (tighter) zero-byte limit.
        """
        if count <= 0:
            return
        if min_bytes_per_element > 0:
            # Only pay for the bytes_remaining() seek on a large block: a small
            # block of positive-size elements must be backed by real bytes on the
            # wire and so cannot over-allocate meaningfully (see
            # _MAX_UNCHECKED_COLLECTION). The structural cap below is always
            # enforced without a seek.
            if count > _MAX_UNCHECKED_COLLECTION:
                remaining = decoder.bytes_remaining()
                # Compare via integer division rather than multiplying, so an
                # attacker-controlled (unbounded) count does not create a huge
                # intermediate product.
                if remaining is not None and count > remaining // min_bytes_per_element:
                    raise avro.errors.InvalidAvroBinaryEncoding(
                        f"Collection claims {count} elements with at least {min_bytes_per_element} bytes each, but only {remaining} bytes are available."
                    )
            if existing + count > structural_limit:
                raise avro.errors.AvroCollectionSizeException(
                    f"Cannot read a collection of more than {structural_limit} elements "
                    f"(declared {existing + count}); raise the {MAX_COLLECTION_ITEMS_ENV} limit if this is legitimate."
                )
            return
        # Zero-byte element type: bound the cumulative count across the datum.
        self._zero_byte_items_read += count
        if self._zero_byte_items_read > zero_byte_limit:
            raise avro.errors.AvroCollectionSizeException(
                f"Cannot read more than {zero_byte_limit} zero-byte collection elements "
                f"in a single datum (reached {self._zero_byte_items_read}); raise the "
                f"{MAX_COLLECTION_ITEMS_ENV} limit if this is legitimate."
            )

    def read_array(self, writers_schema: avro.schema.ArraySchema, readers_schema: avro.schema.ArraySchema, decoder: BinaryDecoder) -> List[object]:
        """
        Arrays are encoded as a series of blocks.

        Each block consists of a long count value,
        followed by that many array items.
        A block with count zero indicates the end of the array.
        Each item is encoded per the array's item schema.

        If a block's count is negative,
        then the count is followed immediately by a long block size,
        indicating the number of bytes in the block.
        The actual count in this case
        is the absolute value of the count written.
        """
        read_items: List[object] = []
        min_bytes = _min_bytes_per_element(writers_schema.items)
        zero_byte_limit, structural_limit = _collection_limits()
        block_count = decoder.read_long()
        while block_count != 0:
            if block_count < 0:
                block_count = -block_count
                decoder.skip_long()
            self._ensure_collection_available(decoder, len(read_items), block_count, min_bytes, zero_byte_limit, structural_limit)
            for i in range(block_count):
                read_items.append(self.read_data(writers_schema.items, readers_schema.items, decoder))
            block_count = decoder.read_long()
        return read_items

    @staticmethod
    def _skip_block_bytes(decoder: BinaryDecoder, block_size: int, block_count: int, min_bytes: int) -> None:
        """Skip a sized-block's byte count, rejecting malformed sizes.

        The block_size is attacker-controlled: a negative value would seek
        backwards and an oversized value past EOF, either corrupting the decoder
        position. Also require that block_size can plausibly hold block_count
        elements at their minimum on-wire size, so a too-small size cannot
        misalign the decoder. A zero-byte element type (``null``, a zero-length
        ``fixed``, or a record of only zero-byte fields) encodes to exactly 0
        bytes, so its block payload must be empty; a positive size would skip
        into the following fields. Reject before skipping.
        """
        if block_size < 0:
            raise avro.errors.InvalidAvroBinaryEncoding(f"Invalid negative block size: {block_size}")
        remaining = decoder.bytes_remaining()
        if remaining is not None and block_size > remaining:
            raise avro.errors.InvalidAvroBinaryEncoding(f"Block size {block_size} exceeds the {remaining} bytes remaining")
        if min_bytes > 0:
            if block_count > block_size // min_bytes:
                raise avro.errors.InvalidAvroBinaryEncoding(
                    f"Block size {block_size} is too small for {block_count} elements of >= {min_bytes} bytes"
                )
        elif block_size != 0:
            raise avro.errors.InvalidAvroBinaryEncoding(f"Block size {block_size} must be 0 for a zero-byte element type")
        try:
            decoder.skip(block_size)
        except (OSError, ValueError, OverflowError, AttributeError, TypeError) as e:
            # The underlying reader can reject the seek (oversized offset, invalid
            # seek, or other IO error), or lack seek()/tell() entirely; surface it
            # as an Avro decoding error rather than leaking a non-Avro exception.
            raise avro.errors.InvalidAvroBinaryEncoding(f"Cannot skip block of {block_size} bytes: {e}") from e

    def skip_array(self, writers_schema: avro.schema.ArraySchema, decoder: BinaryDecoder) -> None:
        min_bytes = _min_bytes_per_element(writers_schema.items)
        zero_byte_limit, structural_limit = _collection_limits()
        items_skipped = 0
        block_count = decoder.read_long()
        while block_count != 0:
            block_size = None
            if block_count < 0:
                block_count = -block_count
                block_size = decoder.read_long()
            # Bound the (normalized) count on both the sized and unsized paths so
            # a negative block count cannot bypass the collection limits.
            self._ensure_collection_available(decoder, items_skipped, block_count, min_bytes, zero_byte_limit, structural_limit)
            items_skipped += block_count
            if block_size is not None:
                self._skip_block_bytes(decoder, block_size, block_count, min_bytes)
            else:
                for i in range(block_count):
                    self.skip_data(writers_schema.items, decoder)
            block_count = decoder.read_long()

    def read_map(self, writers_schema: avro.schema.MapSchema, readers_schema: avro.schema.MapSchema, decoder: BinaryDecoder) -> Mapping[str, object]:
        """
        Maps are encoded as a series of blocks.

        Each block consists of a long count value,
        followed by that many key/value pairs.
        A block with count zero indicates the end of the map.
        Each item is encoded per the map's value schema.

        If a block's count is negative,
        then the count is followed immediately by a long block size,
        indicating the number of bytes in the block.
        The actual count in this case
        is the absolute value of the count written.
        """
        read_items: Dict[str, object] = {}
        # Map keys are strings (>= 1 byte length prefix) plus the value.
        min_bytes = 1 + _min_bytes_per_element(writers_schema.values)
        zero_byte_limit, structural_limit = _collection_limits()
        # Track decoded pairs separately: len(read_items) counts unique keys, so
        # duplicate keys (later entries overwrite earlier ones) would undercount
        # and let a multi-block map exceed the cumulative caps.
        items_read = 0
        block_count = decoder.read_long()
        while block_count != 0:
            if block_count < 0:
                block_count = -block_count
                decoder.skip_long()
            self._ensure_collection_available(decoder, items_read, block_count, min_bytes, zero_byte_limit, structural_limit)
            items_read += block_count
            for i in range(block_count):
                key = decoder.read_utf8()
                read_items[key] = self.read_data(writers_schema.values, readers_schema.values, decoder)
            block_count = decoder.read_long()
        return read_items

    def skip_map(self, writers_schema: avro.schema.MapSchema, decoder: BinaryDecoder) -> None:
        min_bytes = 1 + _min_bytes_per_element(writers_schema.values)
        zero_byte_limit, structural_limit = _collection_limits()
        items_skipped = 0
        block_count = decoder.read_long()
        while block_count != 0:
            block_size = None
            if block_count < 0:
                block_count = -block_count
                block_size = decoder.read_long()
            # Bound the (normalized) count on both the sized and unsized paths so
            # a negative block count cannot bypass the collection limits.
            self._ensure_collection_available(decoder, items_skipped, block_count, min_bytes, zero_byte_limit, structural_limit)
            items_skipped += block_count
            if block_size is not None:
                self._skip_block_bytes(decoder, block_size, block_count, min_bytes)
            else:
                for i in range(block_count):
                    decoder.skip_utf8()
                    self.skip_data(writers_schema.values, decoder)
            block_count = decoder.read_long()

    def read_union(self, writers_schema: avro.schema.UnionSchema, readers_schema: avro.schema.UnionSchema, decoder: BinaryDecoder) -> object:
        """
        A union is encoded by first writing an int value indicating
        the zero-based position within the union of the schema of its value.
        The value is then encoded per the indicated schema within the union.
        """
        # schema resolution
        index_of_schema = int(decoder.read_long())
        if index_of_schema < 0 or index_of_schema >= len(writers_schema.schemas):
            raise avro.errors.SchemaResolutionException(
                f"Can't access branch index {index_of_schema} for union with {len(writers_schema.schemas)} branches", writers_schema, readers_schema
            )
        selected_writers_schema = writers_schema.schemas[index_of_schema]

        # read data
        return self.read_data(selected_writers_schema, readers_schema, decoder)

    def skip_union(self, writers_schema: avro.schema.UnionSchema, decoder: BinaryDecoder) -> None:
        index_of_schema = int(decoder.read_long())
        if index_of_schema < 0 or index_of_schema >= len(writers_schema.schemas):
            raise avro.errors.SchemaResolutionException(
                f"Can't access branch index {index_of_schema} for union with {len(writers_schema.schemas)} branches", writers_schema
            )
        return self.skip_data(writers_schema.schemas[index_of_schema], decoder)

    def read_record(
        self, writers_schema: avro.schema.RecordSchema, readers_schema: avro.schema.RecordSchema, decoder: BinaryDecoder
    ) -> Mapping[str, object]:
        """
        A record is encoded by encoding the values of its fields
        in the order that they are declared. In other words, a record
        is encoded as just the concatenation of the encodings of its fields.
        Field values are encoded per their schema.

        Schema Resolution:
         * the ordering of fields may be different: fields are matched by name.
         * schemas for fields with the same name in both records are resolved
           recursively.
         * if the writer's record contains a field with a name not present in the
           reader's record, the writer's value for that field is ignored.
         * if the reader's record schema has a field that contains a default value,
           and writer's schema does not have a field with the same name, then the
           reader should use the default value from its field.
         * if the reader's record schema has a field with no default value, and
           writer's schema does not have a field with the same name, then the
           field's value is unset.
        """
        # schema resolution
        readers_fields_dict = readers_schema.fields_dict
        read_record = {}
        for field in writers_schema.fields:
            readers_field = readers_fields_dict.get(field.name)
            if readers_field is not None:
                field_val = self.read_data(field.type, readers_field.type, decoder)
                read_record[field.name] = field_val
            else:
                self.skip_data(field.type, decoder)

        # fill in default values
        if len(readers_fields_dict) > len(read_record):
            writers_fields_dict = writers_schema.fields_dict
            for field_name, field in readers_fields_dict.items():
                if field_name not in writers_fields_dict:
                    if not field.has_default:
                        raise avro.errors.SchemaResolutionException(f"No default value for field {field_name}", writers_schema, readers_schema)
                    field_val = self._read_default_value(field.type, field.default)
                    read_record[field.name] = field_val
        return read_record

    def skip_record(self, writers_schema: avro.schema.RecordSchema, decoder: BinaryDecoder) -> None:
        for field in writers_schema.fields:
            self.skip_data(field.type, decoder)

    def _read_default_value(self, field_schema: avro.schema.Schema, default_value: object) -> object:
        """
        Basically a JSON Decoder?
        """
        if field_schema.type == "null":
            if default_value is None:
                return None
            raise avro.errors.InvalidDefaultException(field_schema, default_value)
        if field_schema.type == "boolean":
            return bool(default_value)
        if field_schema.type in ("int", "long"):
            if isinstance(default_value, int):
                return default_value
            raise avro.errors.InvalidDefaultException(field_schema, default_value)
        if field_schema.type in ("float", "double"):
            if isinstance(default_value, float):
                return default_value
            raise avro.errors.InvalidDefaultException(field_schema, default_value)
        if field_schema.type in ("bytes", "fixed"):
            if isinstance(default_value, bytes):
                return default_value
            if isinstance(default_value, str):
                return default_value.encode()
            raise avro.errors.InvalidDefaultException(field_schema, default_value)
        if field_schema.type in ("enum", "string"):
            if isinstance(default_value, str):
                return default_value
            raise avro.errors.InvalidDefaultException(field_schema, default_value)
        if isinstance(field_schema, avro.schema.ArraySchema):
            if isinstance(default_value, Iterable):
                return [self._read_default_value(field_schema.items, json_val) for json_val in default_value]
            raise avro.errors.InvalidDefaultException(field_schema, default_value)
        if isinstance(field_schema, avro.schema.MapSchema):
            if isinstance(default_value, Mapping):
                return {key: self._read_default_value(field_schema.values, json_val) for key, json_val in default_value.items()}
            raise avro.errors.InvalidDefaultException(field_schema, default_value)
        if isinstance(field_schema, avro.schema.UnionSchema):
            return self._read_default_value(field_schema.schemas[0], default_value)
        if isinstance(field_schema, avro.schema.RecordSchema):
            if not isinstance(default_value, Mapping):
                raise avro.errors.InvalidDefaultException(field_schema, default_value)
            read_record = {}
            for field in field_schema.fields:
                json_val = default_value.get(field.name)
                if json_val is None:
                    json_val = field.default
                field_val = self._read_default_value(field.type, json_val)
                read_record[field.name] = field_val
            return read_record
        raise avro.errors.AvroException(f"Unknown type: {field_schema.type}")


class DatumWriter:
    """DatumWriter for generic python objects."""

    _writers_schema: Optional[avro.schema.Schema]

    def __init__(self, writers_schema: Optional[avro.schema.Schema] = None) -> None:
        self._writers_schema = writers_schema

    @property
    def writers_schema(self) -> Optional[avro.schema.Schema]:
        return self._writers_schema

    @writers_schema.setter
    def writers_schema(self, writers_schema: avro.schema.Schema) -> None:
        self._writers_schema = writers_schema

    def write(self, datum: object, encoder: BinaryEncoder) -> None:
        if self.writers_schema is None:
            raise avro.errors.IONotReadyException("Cannot write without a writer's schema.")
        validate(self.writers_schema, datum, raise_on_error=True)
        self.write_data(self.writers_schema, datum, encoder)

    def write_data(self, writers_schema: avro.schema.Schema, datum: object, encoder: BinaryEncoder) -> None:
        # function dispatch to write datum
        logical_type = getattr(writers_schema, "logical_type", None)
        if writers_schema.type == "null":
            if datum is None:
                return encoder.write_null(datum)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if writers_schema.type == "boolean":
            if isinstance(datum, bool):
                return encoder.write_boolean(datum)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if writers_schema.type == "string":
            if isinstance(datum, str):
                return encoder.write_utf8(datum)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if writers_schema.type == "int":
            if logical_type == avro.constants.DATE:
                if isinstance(datum, datetime.date):
                    return encoder.write_date_int(datum)
                warnings.warn(avro.errors.IgnoredLogicalType(f"{datum} is not a date type"))
            elif logical_type == avro.constants.TIME_MILLIS:
                if isinstance(datum, datetime.time):
                    return encoder.write_time_millis_int(datum)
                warnings.warn(avro.errors.IgnoredLogicalType(f"{datum} is not a time type"))
            if isinstance(datum, int):
                return encoder.write_int(datum)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if writers_schema.type == "long":
            if logical_type == avro.constants.TIME_MICROS:
                if isinstance(datum, datetime.time):
                    return encoder.write_time_micros_long(datum)
                warnings.warn(avro.errors.IgnoredLogicalType(f"{datum} is not a time type"))
            elif logical_type == avro.constants.TIMESTAMP_MILLIS:
                if isinstance(datum, datetime.datetime):
                    return encoder.write_timestamp_millis_long(datum)
                warnings.warn(avro.errors.IgnoredLogicalType(f"{datum} is not a datetime type"))
            elif logical_type == avro.constants.TIMESTAMP_MICROS:
                if isinstance(datum, datetime.datetime):
                    return encoder.write_timestamp_micros_long(datum)
                warnings.warn(avro.errors.IgnoredLogicalType(f"{datum} is not a datetime type"))
            if isinstance(datum, int):
                return encoder.write_long(datum)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if writers_schema.type == "float":
            if isinstance(datum, (int, float)):
                return encoder.write_float(datum)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if writers_schema.type == "double":
            if isinstance(datum, (int, float)):
                return encoder.write_double(datum)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if writers_schema.type == "bytes":
            if logical_type == "decimal":
                scale = writers_schema.get_prop("scale")
                if not (isinstance(scale, int) and scale >= 0):
                    warnings.warn(avro.errors.IgnoredLogicalType(f"Invalid decimal scale {scale}. Must be a non-negative integer."))
                elif not isinstance(datum, decimal.Decimal):
                    warnings.warn(avro.errors.IgnoredLogicalType(f"{datum} is not a decimal type"))
                else:
                    return encoder.write_decimal_bytes(datum, scale)
            if isinstance(datum, bytes):
                return encoder.write_bytes(datum)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if isinstance(writers_schema, avro.schema.FixedSchema):
            if logical_type == "decimal":
                scale = writers_schema.get_prop("scale")
                size = writers_schema.size
                if not (isinstance(scale, int) and scale >= 0):
                    warnings.warn(avro.errors.IgnoredLogicalType(f"Invalid decimal scale {scale}. Must be a non-negative integer."))
                elif not isinstance(datum, decimal.Decimal):
                    warnings.warn(avro.errors.IgnoredLogicalType(f"{datum} is not a decimal type"))
                else:
                    return encoder.write_decimal_fixed(datum, scale, size)
            if isinstance(datum, bytes):
                return self.write_fixed(writers_schema, datum, encoder)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if isinstance(writers_schema, avro.schema.EnumSchema):
            if isinstance(datum, str):
                return self.write_enum(writers_schema, datum, encoder)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if isinstance(writers_schema, avro.schema.ArraySchema):
            if isinstance(datum, Sequence):
                return self.write_array(writers_schema, datum, encoder)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if isinstance(writers_schema, avro.schema.MapSchema):
            if isinstance(datum, Mapping):
                return self.write_map(writers_schema, datum, encoder)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        if isinstance(writers_schema, avro.schema.UnionSchema):
            return self.write_union(writers_schema, datum, encoder)
        if isinstance(writers_schema, avro.schema.RecordSchema):
            if isinstance(datum, Mapping):
                return self.write_record(writers_schema, datum, encoder)
            raise avro.errors.AvroTypeException(writers_schema, datum)
        raise avro.errors.AvroException(f"Unknown type: {writers_schema.type}")

    def write_fixed(self, writers_schema: avro.schema.FixedSchema, datum: bytes, encoder: BinaryEncoder) -> None:
        """
        Fixed instances are encoded using the number of bytes declared
        in the schema.
        """
        return encoder.write(datum)

    def write_enum(self, writers_schema: avro.schema.EnumSchema, datum: str, encoder: BinaryEncoder) -> None:
        """
        An enum is encoded by a int, representing the zero-based position
        of the symbol in the schema.
        """
        index_of_datum = writers_schema.symbols.index(datum)
        return encoder.write_int(index_of_datum)

    def write_array(self, writers_schema: avro.schema.ArraySchema, datum: Sequence[object], encoder: BinaryEncoder) -> None:
        """
        Arrays are encoded as a series of blocks.

        Each block consists of a long count value,
        followed by that many array items.
        A block with count zero indicates the end of the array.
        Each item is encoded per the array's item schema.

        If a block's count is negative,
        then the count is followed immediately by a long block size,
        indicating the number of bytes in the block.
        The actual count in this case
        is the absolute value of the count written.
        """
        if len(datum) > 0:
            encoder.write_long(len(datum))
            for item in datum:
                self.write_data(writers_schema.items, item, encoder)
        return encoder.write_long(0)

    def write_map(self, writers_schema: avro.schema.MapSchema, datum: Mapping[str, object], encoder: BinaryEncoder) -> None:
        """
        Maps are encoded as a series of blocks.

        Each block consists of a long count value,
        followed by that many key/value pairs.
        A block with count zero indicates the end of the map.
        Each item is encoded per the map's value schema.

        If a block's count is negative,
        then the count is followed immediately by a long block size,
        indicating the number of bytes in the block.
        The actual count in this case
        is the absolute value of the count written.
        """
        if len(datum) > 0:
            encoder.write_long(len(datum))
            for key, val in datum.items():
                encoder.write_utf8(key)
                self.write_data(writers_schema.values, val, encoder)
        return encoder.write_long(0)

    def write_union(self, writers_schema: avro.schema.UnionSchema, datum: object, encoder: BinaryEncoder) -> None:
        """
        A union is encoded by first writing an int value indicating
        the zero-based position within the union of the schema of its value.
        The value is then encoded per the indicated schema within the union.
        """
        # resolve union
        index_of_schema = -1
        for i, candidate_schema in enumerate(writers_schema.schemas):
            if validate(candidate_schema, datum):
                index_of_schema = i
        if index_of_schema < 0:
            raise avro.errors.AvroTypeException(writers_schema, datum)

        # write data
        encoder.write_long(index_of_schema)
        return self.write_data(writers_schema.schemas[index_of_schema], datum, encoder)

    def write_record(self, writers_schema: avro.schema.RecordSchema, datum: Mapping[str, object], encoder: BinaryEncoder) -> None:
        """
        A record is encoded by encoding the values of its fields
        in the order that they are declared. In other words, a record
        is encoded as just the concatenation of the encodings of its fields.
        Field values are encoded per their schema.
        """
        for field in writers_schema.fields:
            self.write_data(field.type, datum.get(field.name), encoder)
