From 67f6a9c43fc2cced6b82a973b85058a07c25498f Mon Sep 17 00:00:00 2001
From: coder-bot <coder-bot@cloudbison.local>
Date: Mon, 17 Aug 2026 21:54:29 +0000
Subject: [PATCH] WIP checkpoint: Implement chunk streaming system in GDScript

---
 scripts/chunk_data_decoder.gd | 688 ++++++++++++++++++++++++++++++++++
 1 file changed, 688 insertions(+)
 create mode 100644 scripts/chunk_data_decoder.gd

diff --git a/scripts/chunk_data_decoder.gd b/scripts/chunk_data_decoder.gd
new file mode 100644
index 0000000..b490ca2
--- /dev/null
+++ b/scripts/chunk_data_decoder.gd
@@ -0,0 +1,688 @@
+class_name ChunkDataDecoder
+extends RefCounted
+## Decodes chunk binary payloads (and JSON) into logical chunk data.
+##
+## Implements the binary format spec from docs/architecture/world.md §3.6.
+## Both the Godot client and Python backend use this class to ensure
+## identical encoding/decoding behavior.
+##
+## ## Binary Format Overview
+##
+## ```
+## Header (48 bytes):
+##   [0:4]   magic "AGPC"
+##   [4:6]   version (uint16 LE)
+##   [6:8]   header_len (uint16 LE)
+##   [8:12]  flags (uint32 LE)
+##   [12:16] payload_len (uint32 LE)
+##   [16:48] checksum (SHA-256 hex, 32 bytes)
+##
+## Payload Body (sections):
+##   [section_id:1][section_len:4][data:section_len]
+## ```
+
+
+## Content flags parsed from a chunk.
+class ContentFlags:
+	## Severity levels: 0=none, 1=mild, 2=moderate, 3=severe
+	var violence: int = 0
+	var sexual: int = 0
+	var language: int = 0
+	var substance: int = 0
+	var gambling: int = 0
+	var horror: int = 0
+	var hate: int = 0
+	var custom: int = 0
+	var mature: bool = false
+	var sources: Array[String] = []
+
+	func is_mature() -> bool:
+		return mature
+
+	func has_severe_content() -> bool:
+		return (violence >= 3 or sexual >= 3 or language >= 3 or
+		        substance >= 3 or gambling >= 3 or horror >= 3 or
+		        hate >= 3 or custom >= 3)
+
+	func get_max_severity() -> int:
+		return maxi(
+			violence, maxi(sexual, maxi(language, maxi(substance,
+			maxi(gambling, maxi(horror, maxi(hate, custom)))))))
+
+	func to_dict() -> Dictionary:
+		return {
+			"categories": {
+				"violence": violence,
+				"sexual": sexual,
+				"language": language,
+				"substance": substance,
+				"gambling": gambling,
+				"horror": horror,
+				"hate": hate,
+				"custom": custom,
+			},
+			"mature": mature,
+			"sources": sources,
+		}
+
+	func from_dict(d: Dictionary) -> void:
+		if d.has("categories"):
+			violence = d["categories"].get("violence", 0)
+			sexual = d["categories"].get("sexual", 0)
+			language = d["categories"].get("language", 0)
+			substance = d["categories"].get("substance", 0)
+			gambling = d["categories"].get("gambling", 0)
+			horror = d["categories"].get("horror", 0)
+			hate = d["categories"].get("hate", 0)
+			custom = d["categories"].get("custom", 0)
+		mature = d.get("mature", false)
+		sources = d.get("sources", [])
+
+
+## A tile layer definition (palette + dense tile indices).
+class TileLayerData:
+	var name: String = ""
+	var layer_id: int = 0
+	var width: int = 0
+	var height: int = 0
+	var depth: int = 1
+	var palette: Array[TileDef] = []
+	var tiles: PackedInt32Array = PackedInt32Array()
+
+	func tile_count() -> int:
+		return width * height * depth
+
+	func get_tile(x: int, y: int, z: int = 0) -> int:
+		if x < 0 or x >= width or y < 0 or y >= height or z < 0 or z >= depth:
+			return 0  # Out of bounds returns empty tile
+		var idx := (z * height + y) * width + x
+		if idx < tiles.size():
+			return tiles[idx]
+		return 0
+
+	func get_tile_def(tile_id: int) -> TileDef:
+		for def in palette:
+			if def.id == tile_id:
+				return def
+		return null
+
+
+## A single tile definition (palette entry).
+class TileDef:
+	var id: int = 0
+	var asset_id: String = ""
+	var solid: bool = false
+	var flags: int = 0
+	var variants: int = 1
+
+	func is_solid() -> bool:
+		return solid
+
+	func is_walkable() -> bool:
+		return (flags & 0x01) != 0
+
+	func is_climbable() -> bool:
+		return (flags & 0x02) != 0
+
+	func is_liquid() -> bool:
+		return (flags & 0x04) != 0
+
+
+## A world object parsed from chunk data.
+class WorldObjectData:
+	var id: String = ""
+	var type: String = ""
+	var asset_id: String = ""
+	var position: Vector3 = Vector3.ZERO
+	var rotation: Vector4 = Vector4(0, 0, 0, 1)
+	var scale: Vector3 = Vector3.ONE
+	var properties: Dictionary = {}
+	var dynamic: bool = false
+	var flags: ContentFlags = ContentFlags.new()
+
+
+## The complete decoded chunk payload.
+class ChunkPayload:
+	var version: int = 1
+	var key: String = ""
+	var world_id: String = ""
+	var room_id: String = ""
+	var coord: Vector3i = Vector3i.ZERO
+	var bounds: AABB = AABB()
+	var tile_layers: Array[TileLayerData] = []
+	var objects: Array[WorldObjectData] = []
+	var flags: ContentFlags = ContentFlags.new()
+	var created_at: int = 0  # Unix timestamp
+	var updated_at: int = 0  # Unix timestamp
+	var revision: int = 0
+	var author: String = ""
+	var checksum: String = ""
+
+
+# ─── Binary Decoding ────────────────────────────────────────────────────────
+
+static const MAGIC: PackedByteArray = PackedByteArray([0x41, 0x47, 0x50, 0x43])  # "AGPC"
+static const HEADER_SIZE: int = 48
+static const SECTION_META: int = 0x01
+static const SECTION_TILE: int = 0x02
+static const SECTION_OBJECT: int = 0x03
+static const SECTION_FLAGS: int = 0x04
+static const SECTION_REVISION_META: int = 0x05
+
+
+## Decode a binary chunk payload into a ChunkPayload.
+## Returns null if the data is invalid or version is unsupported.
+static func decode_binary(data: PackedByteArray) -> ChunkPayload:
+	if data.size() < HEADER_SIZE:
+		push_error("ChunkDataDecoder: Data too short for header (%d bytes)" % data.size())
+		return null
+
+	# Verify magic
+	var magic := PackedByteArray(data.slice(0, 4))
+	if magic != MAGIC:
+		push_error("ChunkDataDecoder: Invalid magic bytes")
+		return null
+
+	# Read header fields
+	var version := _read_uint16(data, 4)
+	if version != 1:
+		push_error("ChunkDataDecoder: Unsupported chunk version %d" % version)
+		return null
+
+	var header_len := _read_uint16(data, 6)
+	var flags := _read_uint32(data, 8)
+	var payload_len := _read_uint32(data, 12)
+	var checksum := _read_hex_string(data, 16, 32)
+
+	# Verify payload length
+	if data.size() < HEADER_SIZE + payload_len:
+		push_error("ChunkDataDecoder: Payload too short")
+		return null
+
+	var payload := data.slice(HEADER_SIZE, HEADER_SIZE + payload_len)
+
+	# Verify checksum
+	var computed_checksum := _sha256_hex(payload)
+	if computed_checksum != checksum:
+		push_error("ChunkDataDecoder: Checksum mismatch")
+		# Continue anyway for resilience
+
+	var result := ChunkPayload.new()
+	result.version = version
+	result.checksum = checksum
+
+	# Parse sections
+	var offset := 0
+	while offset < payload.size():
+		if offset + 5 > payload.size():
+			break
+
+		var section_id := payload[offset]
+		var section_len := _read_uint32(payload, offset + 1)
+
+		if offset + 5 + section_len > payload.size():
+			break
+
+		var section_data := payload.slice(offset + 5, offset + 5 + section_len)
+
+		match section_id:
+			SECTION_META:
+				_parse_metadata_section(section_data, result)
+			SECTION_TILE:
+				_parse_tile_section(section_data, result)
+			SECTION_OBJECT:
+				_parse_object_section(section_data, result)
+			SECTION_FLAGS:
+				_parse_flags_section(section_data, result.flags)
+			SECTION_REVISION_META:
+				_parse_revision_meta_section(section_data, result)
+
+		offset += 5 + section_len
+
+	return result
+
+
+## Encode a ChunkPayload into binary format.
+static func encode_binary(payload: ChunkPayload) -> PackedByteArray:
+	var sections := PackedByteArray()
+
+	# Build metadata section
+	sections.append_array(_build_metadata_section(payload))
+	# Build tile sections
+	for layer in payload.tile_layers:
+		sections.append_array(_build_tile_section(layer))
+	# Build object sections
+	for obj in payload.objects:
+		sections.append_array(_build_object_section(obj))
+	# Build flags section
+	sections.append_array(_build_flags_section(payload.flags))
+	# Build revision meta section
+	sections.append_array(_build_revision_meta_section(payload))
+
+	var payload_bytes := PackedByteArray(sections)
+	var checksum := _sha256_hex(payload_bytes)
+
+	# Build header
+	var header := PackedByteArray()
+	header.append_array(MAGIC)
+	header.append_array(_write_uint16(1))  # version
+	header.append_array(_write_uint16(0))  # header_len (after this field)
+	header.append_array(_write_uint32(0))  # flags
+	header.append_array(_write_uint32(payload_bytes.size()))
+	header.append_array(_hex_to_bytes(checksum))
+
+	return header + payload_bytes
+
+
+# ─── Section Parsers ────────────────────────────────────────────────────────
+
+static func _parse_metadata_section(data: PackedByteArray, result: ChunkPayload) -> void:
+	var offset := 0
+
+	# world_id (length-prefixed string)
+	var world_id_len := _read_uint16(data, offset)
+	offset += 2
+	result.world_id = data.slice(offset, offset + world_id_len).get_string_from_utf8()
+	offset += world_id_len
+
+	# room_id (length-prefixed string)
+	var room_id_len := _read_uint16(data, offset)
+	offset += 2
+	result.room_id = data.slice(offset, offset + room_id_len).get_string_from_utf8()
+	offset += room_id_len
+
+	# coord (3 × int32)
+	result.coord = Vector3i(
+		_read_int32(data, offset),
+		_read_int32(data, offset + 4),
+		_read_int32(data, offset + 8)
+	)
+	offset += 12
+
+	# bounds (6 × float32)
+	var min_pos := Vector3(
+		_read_float32(data, offset),
+		_read_float32(data, offset + 4),
+		_read_float32(data, offset + 8)
+	)
+	var max_pos := Vector3(
+		_read_float32(data, offset + 12),
+		_read_float32(data, offset + 16),
+		_read_float32(data, offset + 20)
+	)
+	result.bounds = AABB(min_pos, max_pos - min_pos)
+
+
+static func _parse_tile_section(data: PackedByteArray, result: ChunkPayload) -> void:
+	var layer := TileLayerData.new()
+	var offset := 0
+
+	# name (length-prefixed string)
+	var name_len := _read_uint16(data, offset)
+	offset += 2
+	layer.name = data.slice(offset, offset + name_len).get_string_from_utf8()
+	offset += name_len
+
+	# layer_id (uint16)
+	layer.layer_id = _read_uint16(data, offset)
+	offset += 2
+
+	# dimensions (3 × uint16)
+	layer.width = _read_uint16(data, offset)
+	offset += 2
+	layer.height = _read_uint16(data, offset)
+	offset += 2
+	layer.depth = _read_uint16(data, offset)
+	offset += 2
+
+	# palette
+	var palette_count := _read_uint16(data, offset)
+	offset += 2
+	for _i in range(palette_count):
+		var tile_def := TileDef.new()
+		tile_def.id = _read_uint16(data, offset)
+		offset += 2
+
+		var asset_id_len := _read_uint16(data, offset)
+		offset += 2
+		tile_def.asset_id = data.slice(offset, offset + asset_id_len).get_string_from_utf8()
+		offset += asset_id_len
+
+		tile_def.solid = data[offset] != 0
+		offset += 1
+
+		tile_def.flags = _read_uint32(data, offset)
+		offset += 4
+
+		tile_def.variants = data[offset]
+		offset += 1
+
+		layer.palette.append(tile_def)
+
+	# tiles
+	var tile_count := _read_uint32(data, offset)
+	offset += 4
+	for i in range(tile_count):
+		layer.tiles.append(_read_uint16(data, offset))
+		offset += 2
+
+	result.tile_layers.append(layer)
+
+
+static func _parse_object_section(data: PackedByteArray, result: ChunkPayload) -> void:
+	var obj := WorldObjectData.new()
+	var offset := 0
+
+	# id (length-prefixed string)
+	var id_len := _read_uint16(data, offset)
+	offset += 2
+	obj.id = data.slice(offset, offset + id_len).get_string_from_utf8()
+	offset += id_len
+
+	# type (length-prefixed string)
+	var type_len := _read_uint16(data, offset)
+	offset += 2
+	obj.type = data.slice(offset, offset + type_len).get_string_from_utf8()
+	offset += type_len
+
+	# asset_id (length-prefixed string)
+	var asset_id_len := _read_uint16(data, offset)
+	offset += 2
+	obj.asset_id = data.slice(offset, offset + asset_id_len).get_string_from_utf8()
+	offset += asset_id_len
+
+	# position (3 × float32)
+	obj.position = Vector3(
+		_read_float32(data, offset),
+		_read_float32(data, offset + 4),
+		_read_float32(data, offset + 8)
+	)
+	offset += 12
+
+	# rotation (4 × float32)
+	obj.rotation = Vector4(
+		_read_float32(data, offset),
+		_read_float32(data, offset + 4),
+		_read_float32(data, offset + 8),
+		_read_float32(data, offset + 12)
+	)
+	offset += 16
+
+	# scale (3 × float32)
+	obj.scale = Vector3(
+		_read_float32(data, offset),
+		_read_float32(data, offset + 4),
+		_read_float32(data, offset + 8)
+	)
+	offset += 12
+
+	# dynamic (uint8)
+	obj.dynamic = data[offset] != 0
+	offset += 1
+
+	# properties (length-prefixed JSON string)
+	var props_len := _read_uint16(data, offset)
+	offset += 2
+	var props_str := data.slice(offset, offset + props_len).get_string_from_utf8()
+	offset += props_len
+	obj.properties = JSON.parse_string(props_str) or {}
+
+	# flags (ContentFlags)
+	obj.flags.violence = data[offset]
+	offset += 1
+	obj.flags.sexual = data[offset]
+	offset += 1
+	obj.flags.language = data[offset]
+	offset += 1
+	obj.flags.substance = data[offset]
+	offset += 1
+	obj.flags.gambling = data[offset]
+	offset += 1
+	obj.flags.horror = data[offset]
+	offset += 1
+	obj.flags.hate = data[offset]
+	offset += 1
+	obj.flags.custom = data[offset]
+	offset += 1
+	obj.flags.mature = data[offset] != 0
+	offset += 1
+
+	var source_count := _read_uint16(data, offset)
+	offset += 2
+	for _i in range(source_count):
+		var src_len := _read_uint16(data, offset)
+		offset += 2
+		obj.flags.sources.append(data.slice(offset, offset + src_len).get_string_from_utf8())
+		offset += src_len
+
+	result.objects.append(obj)
+
+
+static func _parse_flags_section(data: PackedByteArray, flags: ContentFlags) -> void:
+	flags.violence = data[0]
+	flags.sexual = data[1]
+	flags.language = data[2]
+	flags.substance = data[3]
+	flags.gambling = data[4]
+	flags.horror = data[5]
+	flags.hate = data[6]
+	flags.custom = data[7]
+	flags.mature = data[8] != 0
+
+	var offset := 9
+	var source_count := _read_uint16(data, offset)
+	offset += 2
+	for _i in range(source_count):
+		var src_len := _read_uint16(data, offset)
+		offset += 2
+		flags.sources.append(data.slice(offset, offset + src_len).get_string_from_utf8())
+		offset += src_len
+
+
+static func _parse_revision_meta_section(data: PackedByteArray, result: ChunkPayload) -> void:
+	var offset := 0
+	# created_at (int64 unix)
+	result.created_at = _read_int64(data, offset)
+	offset += 8
+	# updated_at (int64 unix)
+	result.updated_at = _read_int64(data, offset)
+	offset += 8
+	# revision (uint32)
+	result.revision = _read_uint32(data, offset)
+	offset += 4
+	# author (length-prefixed string)
+	var author_len := _read_uint16(data, offset)
+	offset += 2
+	result.author = data.slice(offset, offset + author_len).get_string_from_utf8()
+
+
+# ─── Section Builders ───────────────────────────────────────────────────────
+
+static func _build_metadata_section(payload: ChunkPayload) -> PackedByteArray:
+	var data := PackedByteArray()
+	data.append_array(_write_string(payload.world_id))
+	data.append_array(_write_string(payload.room_id))
+	data.append_array(_write_int32(payload.coord.x))
+	data.append_array(_write_int32(payload.coord.y))
+	data.append_array(_write_int32(payload.coord.z))
+	data.append_array(_write_float32(payload.bounds.position.x))
+	data.append_array(_write_float32(payload.bounds.position.y))
+	data.append_array(_write_float32(payload.bounds.position.z))
+	data.append_array(_write_float32(payload.bounds.size.x))
+	data.append_array(_write_float32(payload.bounds.size.y))
+	data.append_array(_write_float32(payload.bounds.size.z))
+	return data
+
+
+static func _build_tile_section(layer: TileLayerData) -> PackedByteArray:
+	var data := PackedByteArray()
+	data.append_array(_write_string(layer.name))
+	data.append_array(_write_uint16(layer.layer_id))
+	data.append_array(_write_uint16(layer.width))
+	data.append_array(_write_uint16(layer.height))
+	data.append_array(_write_uint16(layer.depth))
+	data.append_array(_write_uint16(layer.palette.size()))
+
+	for tile_def in layer.palette:
+		data.append_array(_write_uint16(tile_def.id))
+		data.append_array(_write_string(tile_def.asset_id))
+		data.append(tile_def.solid as int)
+		data.append_array(_write_uint32(tile_def.flags))
+		data.append(tile_def.variants)
+
+	data.append_array(_write_uint32(layer.tiles.size()))
+	for tile_id in layer.tiles:
+		data.append_array(_write_uint16(tile_id))
+
+	return _wrap_section(data, SECTION_TILE)
+
+
+static func _build_object_section(obj: WorldObjectData) -> PackedByteArray:
+	var data := PackedByteArray()
+	data.append_array(_write_string(obj.id))
+	data.append_array(_write_string(obj.type))
+	data.append_array(_write_string(obj.asset_id))
+	data.append_array(_write_float32(obj.position.x))
+	data.append_array(_write_float32(obj.position.y))
+	data.append_array(_write_float32(obj.position.z))
+	data.append_array(_write_float32(obj.rotation.x))
+	data.append_array(_write_float32(obj.rotation.y))
+	data.append_array(_write_float32(obj.rotation.z))
+	data.append_array(_write_float32(obj.rotation.w))
+	data.append_array(_write_float32(obj.scale.x))
+	data.append_array(_write_float32(obj.scale.y))
+	data.append_array(_write_float32(obj.scale.z))
+	data.append(obj.dynamic as int)
+
+	var props_json := JSON.stringify(obj.properties, "  ")
+	data.append_array(_write_string(props_json))
+
+	data.append(obj.flags.violence)
+	data.append(obj.flags.sexual)
+	data.append(obj.flags.language)
+	data.append(obj.flags.substance)
+	data.append(obj.flags.gambling)
+	data.append(obj.flags.horror)
+	data.append(obj.flags.hate)
+	data.append(obj.flags.custom)
+	data.append(obj.flags.mature as int)
+	data.append_array(_write_uint16(obj.flags.sources.size()))
+	for src in obj.flags.sources:
+		data.append_array(_write_string(src))
+
+	return _wrap_section(data, SECTION_OBJECT)
+
+
+static func _build_flags_section(flags: ContentFlags) -> PackedByteArray:
+	var data := PackedByteArray()
+	data.append(flags.violence)
+	data.append(flags.sexual)
+	data.append(flags.language)
+	data.append(flags.substance)
+	data.append(flags.gambling)
+	data.append(flags.horror)
+	data.append(flags.hate)
+	data.append(flags.custom)
+	data.append(flags.mature as int)
+	data.append_array(_write_uint16(flags.sources.size()))
+	for src in flags.sources:
+		data.append_array(_write_string(src))
+	return _wrap_section(data, SECTION_FLAGS)
+
+
+static func _build_revision_meta_section(payload: ChunkPayload) -> PackedByteArray:
+	var data := PackedByteArray()
+	data.append_array(_write_int64(payload.created_at))
+	data.append_array(_write_int64(payload.updated_at))
+	data.append_array(_write_uint32(payload.revision))
+	data.append_array(_write_string(payload.author))
+	return _wrap_section(data, SECTION_REVISION_META)
+
+
+static func _wrap_section(data: PackedByteArray, section_id: int) -> PackedByteArray:
+	var wrapped := PackedByteArray()
+	wrapped.append(section_id)
+	wrapped.append_array(_write_uint32(data.size()))
+	wrapped.append_array(data)
+	return wrapped
+
+
+# ─── Binary Helpers ─────────────────────────────────────────────────────────
+
+static func _read_uint16(data: PackedByteArray, offset: int) -> int:
+	return data[offset] | (data[offset + 1] << 8)
+
+static func _read_uint32(data: PackedByteArray, offset: int) -> int:
+	return (data[offset] | (data[offset + 1] << 8) |
+	        (data[offset + 2] << 16) | (data[offset + 3] << 24))
+
+static func _read_int32(data: PackedByteArray, offset: int) -> int:
+	var val := _read_uint32(data, offset)
+	if val >= 0x80000000:
+		return val - 0x100000000
+	return val
+
+static func _read_int64(data: PackedByteArray, offset: int) -> int:
+	var lo := _read_uint32(data, offset)
+	var hi := _read_uint32(data, offset + 4)
+	return hi * 0x100000000 + lo
+
+static func _read_float32(data: PackedByteArray, offset: int) -> float:
+	var bits := _read_uint32(data, offset)
+	return bits as float  # GDScript float is 64-bit; bits repr works for IEEE 754
+
+static func _read_hex_string(data: PackedByteArray, offset: int, length: int) -> String:
+	var hex_chars := "0123456789abcdef"
+	var result := ""
+	for i in range(length):
+		var byte_val := data[offset + i]
+		result += hex_chars[byte_val >> 4]
+		result += hex_chars[byte_val & 0x0F]
+	return result
+
+
+static func _write_uint16(val: int) -> PackedByteArray:
+	return PackedByteArray([(val & 0xFF), (val >> 8) & 0xFF])
+
+static func _write_uint32(val: int) -> PackedByteArray:
+	return PackedByteArray([
+		val & 0xFF, (val >> 8) & 0xFF, (val >> 16) & 0xFF, (val >> 24) & 0xFF
+	])
+
+static func _write_int32(val: int) -> PackedByteArray:
+	return _write_uint32(val)
+
+static func _write_int64(val: int) -> PackedByteArray:
+	var lo := val & 0xFFFFFFFF
+	var hi := (val >> 32) & 0xFFFFFFFF
+	return _write_uint32(lo) + _write_uint32(hi)
+
+static func _write_float32(val: float) -> PackedByteArray:
+	var bits := val as int
+	return _write_uint32(bits)
+
+static func _write_string(s: String) -> PackedByteArray:
+	var bytes := s.to_utf8_buffer()
+	return _write_uint16(bytes.size()) + bytes
+
+static func _hex_to_bytes(hex_str: String) -> PackedByteArray:
+	var result := PackedByteArray()
+	for i in range(0, hex_str.length(), 2):
+		var byte_val := int("0x" + hex_str.substr(i, 2), 16)
+		result.append(byte_val)
+	return result
+
+
+# ─── SHA-256 ────────────────────────────────────────────────────────────────
+
+static func _sha256_hex(data: PackedByteArray) -> String:
+	var sha256 := SHA256Hash.new()
+	sha256.start()
+	sha256.update(data)
+	var hash := sha256.finish()
+	var hex_chars := "0123456789abcdef"
+	var result := ""
+	for byte_val in hash:
+		result += hex_chars[(byte_val >> 4) & 0x0F]
+		result += hex_chars[byte_val & 0x0F]
+	return result