rewrite packet parser as string-based recursive descent

internal/packets/builder.go

@@ -0,0 +1,454 @@
+package packets
+
+import (
+	"bytes"
+	"encoding/binary"
+	"eq2emu/internal/common"
+	"eq2emu/internal/packets/parser"
+	"fmt"
+	"math"
+)
+
+// PacketBuilder constructs packets from data using parsed packet definitions
+type PacketBuilder struct {
+	def     *parser.PacketDef
+	version uint32
+	flags   uint64
+	buf     *bytes.Buffer
+}
+
+// NewPacketBuilder creates a new packet builder for the given packet definition
+func NewPacketBuilder(def *parser.PacketDef, version uint32, flags uint64) *PacketBuilder {
+	return &PacketBuilder{
+		def:     def,
+		version: version,
+		flags:   flags,
+		buf:     new(bytes.Buffer),
+	}
+}
+
+// Build constructs a packet from the provided data map
+func (b *PacketBuilder) Build(data map[string]any) ([]byte, error) {
+	b.buf.Reset()
+
+	err := b.buildStruct(data, b.def)
+	if err != nil {
+		return nil, err
+	}
+
+	return b.buf.Bytes(), nil
+}
+
+// buildStruct builds a struct according to the packet definition field order
+func (b *PacketBuilder) buildStruct(data map[string]any, def *parser.PacketDef) error {
+	order := b.getVersionOrder(def)
+
+	for _, fieldName := range order {
+		field, exists := def.Fields[fieldName]
+		if !exists {
+			continue
+		}
+
+		// Skip fields based on conditions
+		if !b.checkCondition(field.Condition, data) {
+			continue
+		}
+
+		fieldType := field.Type
+		if field.Type2 != 0 && b.checkCondition(field.Type2Cond, data) {
+			fieldType = field.Type2
+		}
+
+		value, hasValue := data[fieldName]
+		if !hasValue && !field.Optional {
+			return fmt.Errorf("required field '%s' not found in data", fieldName)
+		}
+
+		if hasValue {
+			err := b.buildField(value, field, fieldType, fieldName)
+			if err != nil {
+				return fmt.Errorf("error building field '%s': %w", fieldName, err)
+			}
+		} else if field.Optional {
+			// Write default value for optional fields
+			err := b.writeDefaultValue(field, fieldType)
+			if err != nil {
+				return fmt.Errorf("error writing default value for field '%s': %w", fieldName, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// buildField writes a single field to the packet buffer
+func (b *PacketBuilder) buildField(value any, field parser.FieldDesc, fieldType common.EQ2DataType, fieldName string) error {
+	switch fieldType {
+	case common.TypeInt8:
+		v, ok := value.(uint8)
+		if !ok {
+			return fmt.Errorf("field '%s' expected uint8, got %T", fieldName, value)
+		}
+		if field.Oversized > 0 {
+			return b.writeOversizedUint8(v, field.Oversized)
+		}
+		return binary.Write(b.buf, binary.LittleEndian, v)
+
+	case common.TypeInt16:
+		v, ok := value.(uint16)
+		if !ok {
+			return fmt.Errorf("field '%s' expected uint16, got %T", fieldName, value)
+		}
+		if field.Oversized > 0 {
+			return b.writeOversizedUint16(v, field.Oversized)
+		}
+		return binary.Write(b.buf, binary.LittleEndian, v)
+
+	case common.TypeInt32:
+		v, ok := value.(uint32)
+		if !ok {
+			return fmt.Errorf("field '%s' expected uint32, got %T", fieldName, value)
+		}
+		if field.Oversized > 0 {
+			return b.writeOversizedUint32(v, field.Oversized)
+		}
+		return binary.Write(b.buf, binary.LittleEndian, v)
+
+	case common.TypeInt64:
+		v, ok := value.(uint64)
+		if !ok {
+			return fmt.Errorf("field '%s' expected uint64, got %T", fieldName, value)
+		}
+		return binary.Write(b.buf, binary.LittleEndian, v)
+
+	case common.TypeSInt8:
+		v, ok := value.(int8)
+		if !ok {
+			return fmt.Errorf("field '%s' expected int8, got %T", fieldName, value)
+		}
+		return binary.Write(b.buf, binary.LittleEndian, v)
+
+	case common.TypeSInt16:
+		v, ok := value.(int16)
+		if !ok {
+			return fmt.Errorf("field '%s' expected int16, got %T", fieldName, value)
+		}
+		if field.Oversized > 0 {
+			return b.writeOversizedSint16(v, field.Oversized)
+		}
+		return binary.Write(b.buf, binary.LittleEndian, v)
+
+	case common.TypeSInt32:
+		v, ok := value.(int32)
+		if !ok {
+			return fmt.Errorf("field '%s' expected int32, got %T", fieldName, value)
+		}
+		return binary.Write(b.buf, binary.LittleEndian, v)
+
+	case common.TypeSInt64:
+		v, ok := value.(int64)
+		if !ok {
+			return fmt.Errorf("field '%s' expected int64, got %T", fieldName, value)
+		}
+		return binary.Write(b.buf, binary.LittleEndian, v)
+
+	case common.TypeString8:
+		v, ok := value.(string)
+		if !ok {
+			return fmt.Errorf("field '%s' expected string, got %T", fieldName, value)
+		}
+		return b.writeEQ2String8(v)
+
+	case common.TypeString16:
+		v, ok := value.(string)
+		if !ok {
+			return fmt.Errorf("field '%s' expected string, got %T", fieldName, value)
+		}
+		return b.writeEQ2String16(v)
+
+	case common.TypeString32:
+		v, ok := value.(string)
+		if !ok {
+			return fmt.Errorf("field '%s' expected string, got %T", fieldName, value)
+		}
+		return b.writeEQ2String32(v)
+
+	case common.TypeChar:
+		v, ok := value.([]byte)
+		if !ok {
+			return fmt.Errorf("field '%s' expected []byte, got %T", fieldName, value)
+		}
+		return b.writeBytes(v, field.Length)
+
+	case common.TypeFloat:
+		v, ok := value.(float32)
+		if !ok {
+			return fmt.Errorf("field '%s' expected float32, got %T", fieldName, value)
+		}
+		return binary.Write(b.buf, binary.LittleEndian, v)
+
+	case common.TypeDouble:
+		v, ok := value.(float64)
+		if !ok {
+			return fmt.Errorf("field '%s' expected float64, got %T", fieldName, value)
+		}
+		return binary.Write(b.buf, binary.LittleEndian, v)
+
+	case common.TypeColor:
+		v, ok := value.(common.EQ2Color)
+		if !ok {
+			return fmt.Errorf("field '%s' expected EQ2Color, got %T", fieldName, value)
+		}
+		return b.writeEQ2Color(v)
+
+	case common.TypeEquipment:
+		v, ok := value.(common.EQ2EquipmentItem)
+		if !ok {
+			return fmt.Errorf("field '%s' expected EQ2EquipmentItem, got %T", fieldName, value)
+		}
+		return b.writeEQ2Equipment(v)
+
+	case common.TypeArray:
+		v, ok := value.([]map[string]any)
+		if !ok {
+			return fmt.Errorf("field '%s' expected []map[string]any, got %T", fieldName, value)
+		}
+		return b.buildArray(v, field)
+
+	default:
+		return fmt.Errorf("unsupported field type %d for field '%s'", fieldType, fieldName)
+	}
+}
+
+// buildArray builds an array field
+func (b *PacketBuilder) buildArray(items []map[string]any, field parser.FieldDesc) error {
+	if field.SubDef == nil {
+		return fmt.Errorf("array field missing sub-definition")
+	}
+
+	size := len(items)
+	if field.MaxArraySize > 0 && size > field.MaxArraySize {
+		size = field.MaxArraySize
+	}
+
+	for i := 0; i < size; i++ {
+		err := b.buildStruct(items[i], field.SubDef)
+		if err != nil {
+			return fmt.Errorf("error building array item %d: %w", i, err)
+		}
+	}
+
+	return nil
+}
+
+// Helper methods for writing specific data types
+func (b *PacketBuilder) writeEQ2String8(s string) error {
+	data := []byte(s)
+	size := len(data)
+	if size > math.MaxUint8 {
+		size = math.MaxUint8
+	}
+
+	err := binary.Write(b.buf, binary.LittleEndian, uint8(size))
+	if err != nil {
+		return err
+	}
+
+	_, err = b.buf.Write(data[:size])
+	return err
+}
+
+func (b *PacketBuilder) writeEQ2String16(s string) error {
+	data := []byte(s)
+	size := len(data)
+	if size > math.MaxUint16 {
+		size = math.MaxUint16
+	}
+
+	err := binary.Write(b.buf, binary.LittleEndian, uint16(size))
+	if err != nil {
+		return err
+	}
+
+	_, err = b.buf.Write(data[:size])
+	return err
+}
+
+func (b *PacketBuilder) writeEQ2String32(s string) error {
+	data := []byte(s)
+	size := len(data)
+	if size > math.MaxUint32 {
+		size = math.MaxUint32
+	}
+
+	err := binary.Write(b.buf, binary.LittleEndian, uint32(size))
+	if err != nil {
+		return err
+	}
+
+	_, err = b.buf.Write(data[:size])
+	return err
+}
+
+func (b *PacketBuilder) writeBytes(data []byte, length int) error {
+	if length > 0 && len(data) > length {
+		data = data[:length]
+	}
+
+	_, err := b.buf.Write(data)
+	return err
+}
+
+func (b *PacketBuilder) writeEQ2Color(color common.EQ2Color) error {
+	err := binary.Write(b.buf, binary.LittleEndian, color.Red)
+	if err != nil {
+		return err
+	}
+	err = binary.Write(b.buf, binary.LittleEndian, color.Green)
+	if err != nil {
+		return err
+	}
+	return binary.Write(b.buf, binary.LittleEndian, color.Blue)
+}
+
+func (b *PacketBuilder) writeEQ2Equipment(item common.EQ2EquipmentItem) error {
+	err := binary.Write(b.buf, binary.LittleEndian, item.Type)
+	if err != nil {
+		return err
+	}
+	err = b.writeEQ2Color(item.Color)
+	if err != nil {
+		return err
+	}
+	return b.writeEQ2Color(item.Highlight)
+}
+
+func (b *PacketBuilder) writeOversizedUint8(value uint8, threshold int) error {
+	if int(value) >= threshold {
+		err := binary.Write(b.buf, binary.LittleEndian, uint8(255))
+		if err != nil {
+			return err
+		}
+		return binary.Write(b.buf, binary.LittleEndian, uint16(value))
+	}
+	return binary.Write(b.buf, binary.LittleEndian, value)
+}
+
+func (b *PacketBuilder) writeOversizedUint16(value uint16, threshold int) error {
+	if int(value) >= threshold {
+		err := binary.Write(b.buf, binary.LittleEndian, uint16(65535))
+		if err != nil {
+			return err
+		}
+		return binary.Write(b.buf, binary.LittleEndian, uint32(value))
+	}
+	return binary.Write(b.buf, binary.LittleEndian, value)
+}
+
+func (b *PacketBuilder) writeOversizedUint32(value uint32, threshold int) error {
+	if int64(value) >= int64(threshold) {
+		err := binary.Write(b.buf, binary.LittleEndian, uint32(4294967295))
+		if err != nil {
+			return err
+		}
+		return binary.Write(b.buf, binary.LittleEndian, uint64(value))
+	}
+	return binary.Write(b.buf, binary.LittleEndian, value)
+}
+
+func (b *PacketBuilder) writeOversizedSint16(value int16, threshold int) error {
+	if int(value) >= threshold {
+		err := binary.Write(b.buf, binary.LittleEndian, int16(-1))
+		if err != nil {
+			return err
+		}
+		return binary.Write(b.buf, binary.LittleEndian, int32(value))
+	}
+	return binary.Write(b.buf, binary.LittleEndian, value)
+}
+
+func (b *PacketBuilder) writeDefaultValue(field parser.FieldDesc, fieldType common.EQ2DataType) error {
+	switch fieldType {
+	case common.TypeInt8:
+		return binary.Write(b.buf, binary.LittleEndian, uint8(field.DefaultValue))
+	case common.TypeInt16:
+		return binary.Write(b.buf, binary.LittleEndian, uint16(field.DefaultValue))
+	case common.TypeInt32:
+		return binary.Write(b.buf, binary.LittleEndian, uint32(field.DefaultValue))
+	case common.TypeInt64:
+		return binary.Write(b.buf, binary.LittleEndian, uint64(field.DefaultValue))
+	case common.TypeSInt8:
+		return binary.Write(b.buf, binary.LittleEndian, field.DefaultValue)
+	case common.TypeSInt16:
+		return binary.Write(b.buf, binary.LittleEndian, int16(field.DefaultValue))
+	case common.TypeSInt32:
+		return binary.Write(b.buf, binary.LittleEndian, int32(field.DefaultValue))
+	case common.TypeSInt64:
+		return binary.Write(b.buf, binary.LittleEndian, int64(field.DefaultValue))
+	case common.TypeString8:
+		return b.writeEQ2String8("")
+	case common.TypeString16:
+		return b.writeEQ2String16("")
+	case common.TypeString32:
+		return b.writeEQ2String32("")
+	case common.TypeFloat:
+		return binary.Write(b.buf, binary.LittleEndian, float32(field.DefaultValue))
+	case common.TypeDouble:
+		return binary.Write(b.buf, binary.LittleEndian, float64(field.DefaultValue))
+	case common.TypeColor:
+		color := common.EQ2Color{Red: field.DefaultValue, Green: field.DefaultValue, Blue: field.DefaultValue}
+		return b.writeEQ2Color(color)
+	case common.TypeChar:
+		if field.Length > 0 {
+			defaultBytes := make([]byte, field.Length)
+			return b.writeBytes(defaultBytes, field.Length)
+		}
+	}
+	return nil
+}
+
+func (b *PacketBuilder) getVersionOrder(def *parser.PacketDef) []string {
+	var bestVersion uint32
+	for v := range def.Orders {
+		if v <= b.version && v > bestVersion {
+			bestVersion = v
+		}
+	}
+	return def.Orders[bestVersion]
+}
+
+func (b *PacketBuilder) checkCondition(condition string, data map[string]any) bool {
+	if condition == "" {
+		return true
+	}
+
+	// Simple condition evaluation - this would need to be expanded
+	// for complex expressions used in the packet definitions
+	if value, exists := data[condition]; exists {
+		switch v := value.(type) {
+		case bool:
+			return v
+		case int, int8, int16, int32, int64:
+			return v != 0
+		case uint, uint8, uint16, uint32, uint64:
+			return v != 0
+		case string:
+			return v != ""
+		default:
+			return true
+		}
+	}
+
+	return false
+}
+
+// BuildPacket is a convenience function that creates a builder and builds a packet in one call
+func BuildPacket(packetName string, data map[string]any, version uint32, flags uint64) ([]byte, error) {
+	def, exists := GetPacket(packetName)
+	if !exists {
+		return nil, fmt.Errorf("packet definition '%s' not found", packetName)
+	}
+
+	builder := NewPacketBuilder(def, version, flags)
+	return builder.Build(data)
+}

internal/packets/builder_test.go

@@ -0,0 +1,173 @@
+package packets
+
+import (
+	"eq2emu/internal/common"
+	"eq2emu/internal/packets/parser"
+	"testing"
+)
+
+func TestPacketBuilderSimpleFields(t *testing.T) {
+	// Create a simple packet definition for testing
+	def := parser.NewPacketDef(5)
+
+	// Add some basic fields
+	def.Fields["field1"] = parser.FieldDesc{
+		Type: common.TypeInt8,
+	}
+	def.Fields["field2"] = parser.FieldDesc{
+		Type: common.TypeString8,
+	}
+	def.Fields["field3"] = parser.FieldDesc{
+		Type: common.TypeInt32,
+	}
+
+	// Set field order
+	def.Orders[1] = []string{"field1", "field2", "field3"}
+
+	// Create test data
+	data := map[string]any{
+		"field1": uint8(42),
+		"field2": "hello",
+		"field3": uint32(12345),
+	}
+
+	// Build packet
+	builder := NewPacketBuilder(def, 1, 0)
+	packetData, err := builder.Build(data)
+	if err != nil {
+		t.Errorf("Failed to build simple packet: %v", err)
+		return
+	}
+
+	expectedMinLength := 1 + 1 + 5 + 4 // uint8 + string8(len+data) + uint32
+	if len(packetData) < expectedMinLength {
+		t.Errorf("Built packet too short: got %d bytes, expected at least %d", len(packetData), expectedMinLength)
+		return
+	}
+
+	t.Logf("Successfully built simple packet (%d bytes)", len(packetData))
+
+	// Verify the first byte is correct
+	if packetData[0] != 42 {
+		t.Errorf("First field incorrect: got %d, expected 42", packetData[0])
+	}
+}
+
+func TestPacketBuilderOptionalFields(t *testing.T) {
+	// Create a packet definition with optional fields
+	def := parser.NewPacketDef(3)
+
+	def.Fields["required"] = parser.FieldDesc{
+		Type: common.TypeInt8,
+	}
+	def.Fields["optional"] = parser.FieldDesc{
+		Type:     common.TypeInt8,
+		Optional: true,
+	}
+
+	def.Orders[1] = []string{"required", "optional"}
+
+	// Test with only required field
+	data := map[string]any{
+		"required": uint8(123),
+	}
+
+	builder := NewPacketBuilder(def, 1, 0)
+	packetData, err := builder.Build(data)
+	if err != nil {
+		t.Errorf("Failed to build packet with optional fields: %v", err)
+		return
+	}
+
+	expectedLength := 2 // required + optional default
+	if len(packetData) != expectedLength {
+		t.Errorf("Built packet wrong length: got %d bytes, expected %d", len(packetData), expectedLength)
+		return
+	}
+
+	t.Logf("Successfully built packet with optional fields (%d bytes)", len(packetData))
+}
+
+func TestPacketBuilderMissingRequiredField(t *testing.T) {
+	// Create a packet definition
+	def := parser.NewPacketDef(2)
+
+	def.Fields["required"] = parser.FieldDesc{
+		Type: common.TypeInt8,
+	}
+	def.Fields["also_required"] = parser.FieldDesc{
+		Type: common.TypeInt16,
+	}
+
+	def.Orders[1] = []string{"required", "also_required"}
+
+	// Test with missing required field
+	data := map[string]any{
+		"required": uint8(123),
+		// missing "also_required"
+	}
+
+	builder := NewPacketBuilder(def, 1, 0)
+	_, err := builder.Build(data)
+	if err == nil {
+		t.Error("Expected error for missing required field, but got none")
+		return
+	}
+
+	t.Logf("Correctly detected missing required field: %v", err)
+}
+
+func TestPacketBuilderStringTypes(t *testing.T) {
+	// Test different string types
+	def := parser.NewPacketDef(3)
+
+	def.Fields["str8"] = parser.FieldDesc{
+		Type: common.TypeString8,
+	}
+	def.Fields["str16"] = parser.FieldDesc{
+		Type: common.TypeString16,
+	}
+	def.Fields["str32"] = parser.FieldDesc{
+		Type: common.TypeString32,
+	}
+
+	def.Orders[1] = []string{"str8", "str16", "str32"}
+
+	data := map[string]any{
+		"str8":  "test8",
+		"str16": "test16",
+		"str32": "test32",
+	}
+
+	builder := NewPacketBuilder(def, 1, 0)
+	packetData, err := builder.Build(data)
+	if err != nil {
+		t.Errorf("Failed to build packet with strings: %v", err)
+		return
+	}
+
+	// Verify we have some data
+	if len(packetData) < 20 { // rough estimate
+		t.Errorf("Built packet too short for string data: %d bytes", len(packetData))
+		return
+	}
+
+	t.Logf("Successfully built packet with strings (%d bytes)", len(packetData))
+}
+
+func TestBuildPacketConvenienceFunction(t *testing.T) {
+	// Test the convenience function
+	data := map[string]any{
+		"username": "testuser",
+		"password": "testpass",
+	}
+
+	// This should fail since we don't have a "TestPacket" defined
+	_, err := BuildPacket("NonExistentPacket", data, 1, 0)
+	if err == nil {
+		t.Error("Expected error for non-existent packet, but got none")
+		return
+	}
+
+	t.Logf("Correctly detected non-existent packet: %v", err)
+}

internal/packets/loader.go

@@ -78,7 +78,7 @@ func processDirectory(dirPath string, packets map[string]*parser.PacketDef) erro
 
 		err := processXMLFile(entryPath, packets)
 		if err != nil {
-			log.Printf("Warning: failed to process %s: %v", entryPath, err)
+			log.Printf("Warning: %s: %v", entryPath, err)
 		}
 	}
 
@@ -93,7 +93,7 @@ func processXMLFile(filePath string, packets map[string]*parser.PacketDef) error
 
 	parsedPackets, err := parser.Parse(string(content))
 	if err != nil {
-		return fmt.Errorf("failed to parse XML: %w", err)
+		return fmt.Errorf("failed to parse packet def: %w", err)
 	}
 
 	for name, packet := range parsedPackets {

internal/packets/parser/lexer.go

@@ -1,358 +0,0 @@
-package parser
-
-import (
-	"fmt"
-	"sync"
-	"unicode"
-)
-
-// Object pools for heavy reuse
-var tokenPool = sync.Pool{
-	New: func() any {
-		return &Token{
-			Attributes: make(map[string]string, 8),
-			TagStart:   -1,
-			TagEnd:     -1,
-			TextStart:  -1,
-			TextEnd:    -1,
-		}
-	},
-}
-
-// More efficient lexer using byte operations and minimal allocations
-type Lexer struct {
-	input []byte // Use byte slice for faster operations
-	pos   int
-	line  int
-	col   int
-}
-
-// Creates a new lexer
-func NewLexer(input string) *Lexer {
-	return &Lexer{
-		input: []byte(input),
-		line:  1,
-		col:   1,
-	}
-}
-
-// Returns next byte without advancing
-func (l *Lexer) peek() byte {
-	if l.pos >= len(l.input) {
-		return 0
-	}
-	return l.input[l.pos]
-}
-
-// Advances and returns next byte
-func (l *Lexer) next() byte {
-	if l.pos >= len(l.input) {
-		return 0
-	}
-	ch := l.input[l.pos]
-	l.pos++
-	if ch == '\n' {
-		l.line++
-		l.col = 1
-	} else {
-		l.col++
-	}
-	return ch
-}
-
-// Checks if a tag should be treated as self-closing (using byte comparison)
-func (l *Lexer) isSelfClosingTag(start, end int) bool {
-	length := end - start
-	if length < 2 || length > 6 {
-		return false
-	}
-
-	// Fast byte-based comparison
-	switch length {
-	case 2:
-		return (l.input[start] == 'i' && l.input[start+1] == '8') ||
-			(l.input[start] == 'f' && l.input[start+1] == '2')
-	case 3:
-		return (l.input[start] == 'i' && l.input[start+1] == '1' && l.input[start+2] == '6') ||
-			(l.input[start] == 'i' && l.input[start+1] == '3' && l.input[start+2] == '2') ||
-			(l.input[start] == 'i' && l.input[start+1] == '6' && l.input[start+2] == '4') ||
-			(l.input[start] == 's' && l.input[start+1] == 'i' && l.input[start+2] == '8') ||
-			(l.input[start] == 'f' && l.input[start+1] == '3' && l.input[start+2] == '2') ||
-			(l.input[start] == 'f' && l.input[start+1] == '6' && l.input[start+2] == '4')
-	case 4:
-		return (l.input[start] == 's' && l.input[start+1] == 'i' &&
-			l.input[start+2] == '1' && l.input[start+3] == '6') ||
-			(l.input[start] == 's' && l.input[start+1] == 'i' &&
-				l.input[start+2] == '3' && l.input[start+3] == '2') ||
-			(l.input[start] == 's' && l.input[start+1] == 'i' &&
-				l.input[start+2] == '6' && l.input[start+3] == '4') ||
-			(l.input[start] == 'c' && l.input[start+1] == 'h' &&
-				l.input[start+2] == 'a' && l.input[start+3] == 'r') ||
-			(l.input[start] == 's' && l.input[start+1] == 't' &&
-				l.input[start+2] == 'r' && l.input[start+3] == '8')
-	case 5:
-		return (l.input[start] == 'c' && l.input[start+1] == 'o' &&
-			l.input[start+2] == 'l' && l.input[start+3] == 'o' &&
-			l.input[start+4] == 'r') ||
-			(l.input[start] == 'e' && l.input[start+1] == 'q' &&
-				l.input[start+2] == 'u' && l.input[start+3] == 'i' &&
-				l.input[start+4] == 'p') ||
-			(l.input[start] == 's' && l.input[start+1] == 't' &&
-				l.input[start+2] == 'r' && l.input[start+3] == '1' &&
-				l.input[start+4] == '6') ||
-			(l.input[start] == 's' && l.input[start+1] == 't' &&
-				l.input[start+2] == 'r' && l.input[start+3] == '3' &&
-				l.input[start+4] == '2')
-	case 6:
-		return (l.input[start] == 'd' && l.input[start+1] == 'o' &&
-			l.input[start+2] == 'u' && l.input[start+3] == 'b' &&
-			l.input[start+4] == 'l' && l.input[start+5] == 'e')
-	}
-	return false
-}
-
-// Skips whitespace using byte operations
-func (l *Lexer) skipWhitespace() {
-	for l.pos < len(l.input) {
-		ch := l.input[l.pos]
-		if ch == ' ' || ch == '\t' || ch == '\n' || ch == '\r' {
-			if ch == '\n' {
-				l.line++
-				l.col = 1
-			} else {
-				l.col++
-			}
-			l.pos++
-		} else {
-			break
-		}
-	}
-}
-
-// Optimized attribute parsing with minimal allocations - FIXED BUG
-func (l *Lexer) parseAttributes(attrs map[string]string) error {
-	// Clear existing attributes without deallocating
-	for k := range attrs {
-		delete(attrs, k)
-	}
-
-	for {
-		l.skipWhitespace()
-		if l.pos >= len(l.input) || l.peek() == '>' ||
-			(l.peek() == '/' && l.pos+1 < len(l.input) && l.input[l.pos+1] == '>') {
-			break
-		}
-
-		// Read attribute name using byte operations
-		nameStart := l.pos
-		for l.pos < len(l.input) {
-			ch := l.input[l.pos]
-			if ch == '=' || ch == ' ' || ch == '\t' || ch == '\n' || ch == '>' {
-				break
-			}
-			l.pos++
-			if ch != '\n' {
-				l.col++
-			}
-		}
-
-		nameEnd := l.pos // FIXED: Store end of name here
-
-		if nameStart == nameEnd {
-			break
-		}
-
-		l.skipWhitespace()
-		if l.peek() != '=' {
-			return fmt.Errorf("expected '=' after attribute name")
-		}
-		l.next() // skip '='
-		l.skipWhitespace()
-
-		// Read attribute value
-		quote := l.peek()
-		if quote != '"' && quote != '\'' {
-			return fmt.Errorf("attribute value must be quoted")
-		}
-		l.next() // skip opening quote
-
-		valueStart := l.pos
-		for l.pos < len(l.input) && l.input[l.pos] != quote {
-			if l.input[l.pos] == '\n' {
-				l.line++
-				l.col = 1
-			} else {
-				l.col++
-			}
-			l.pos++
-		}
-
-		if l.pos >= len(l.input) {
-			return fmt.Errorf("unclosed attribute value")
-		}
-
-		// FIXED: Correct name and value extraction
-		name := string(l.input[nameStart:nameEnd])
-		value := string(l.input[valueStart:l.pos])
-		attrs[name] = value
-
-		l.next() // skip closing quote
-	}
-
-	return nil
-}
-
-// Optimized token generation with pooling
-func (l *Lexer) NextToken() *Token {
-	token := tokenPool.Get().(*Token)
-	token.Type = TokenError
-	token.TagStart = -1
-	token.TagEnd = -1
-	token.TextStart = -1
-	token.TextEnd = -1
-	token.Line = l.line
-	token.Col = l.col
-
-	l.skipWhitespace()
-	if l.pos >= len(l.input) {
-		token.Type = TokenEOF
-		return token
-	}
-
-	if l.peek() == '<' {
-		l.next() // skip '<'
-
-		// Check for comment using byte comparison
-		if l.pos+2 < len(l.input) &&
-			l.input[l.pos] == '!' && l.input[l.pos+1] == '-' && l.input[l.pos+2] == '-' {
-			l.pos += 3
-			start := l.pos
-			// Find end of comment efficiently
-			for l.pos+2 < len(l.input) {
-				if l.input[l.pos] == '-' && l.input[l.pos+1] == '-' && l.input[l.pos+2] == '>' {
-					token.Type = TokenComment
-					token.TextStart = start
-					token.TextEnd = l.pos
-					l.pos += 3
-					return token
-				}
-				if l.input[l.pos] == '\n' {
-					l.line++
-					l.col = 1
-				} else {
-					l.col++
-				}
-				l.pos++
-			}
-			token.Type = TokenError
-			return token
-		}
-
-		// Check for closing tag
-		if l.peek() == '/' {
-			l.next() // skip '/'
-			start := l.pos
-			for l.pos < len(l.input) && l.input[l.pos] != '>' {
-				l.pos++
-				l.col++
-			}
-			if l.pos >= len(l.input) {
-				token.Type = TokenError
-				return token
-			}
-			token.Type = TokenCloseTag
-			token.TagStart = start
-			token.TagEnd = l.pos
-			l.next() // skip '>'
-			return token
-		}
-
-		// Opening or self-closing tag
-		start := l.pos
-		for l.pos < len(l.input) {
-			ch := l.input[l.pos]
-			if ch == '>' || ch == '/' || ch == ' ' || ch == '\t' || ch == '\n' {
-				break
-			}
-			l.pos++
-			l.col++
-		}
-
-		if start == l.pos {
-			token.Type = TokenError
-			return token
-		}
-
-		token.TagStart = start
-		token.TagEnd = l.pos
-
-		if err := l.parseAttributes(token.Attributes); err != nil {
-			token.Type = TokenError
-			return token
-		}
-
-		l.skipWhitespace()
-		if l.pos >= len(l.input) {
-			token.Type = TokenError
-			return token
-		}
-
-		if l.peek() == '/' && l.pos+1 < len(l.input) && l.input[l.pos+1] == '>' {
-			token.Type = TokenSelfCloseTag
-			l.pos += 2
-		} else {
-			// Check if this is a self-closing field type
-			if l.isSelfClosingTag(token.TagStart, token.TagEnd) {
-				token.Type = TokenSelfCloseTag
-			} else {
-				token.Type = TokenOpenTag
-			}
-			if l.peek() == '>' {
-				l.next()
-			} else {
-				token.Type = TokenError
-				return token
-			}
-		}
-
-		return token
-	}
-
-	// Text content - find range without copying
-	start := l.pos
-	for l.pos < len(l.input) && l.input[l.pos] != '<' {
-		if l.input[l.pos] == '\n' {
-			l.line++
-			l.col = 1
-		} else {
-			l.col++
-		}
-		l.pos++
-	}
-
-	// Trim whitespace from range
-	for start < l.pos && unicode.IsSpace(rune(l.input[start])) {
-		start++
-	}
-	end := l.pos
-	for end > start && unicode.IsSpace(rune(l.input[end-1])) {
-		end--
-	}
-
-	if start < end {
-		token.Type = TokenText
-		token.TextStart = start
-		token.TextEnd = end
-		return token
-	}
-
-	// Skip empty text, get next token
-	return l.NextToken()
-}
-
-// Returns token to pool
-func (l *Lexer) ReleaseToken(token *Token) {
-	if token != nil {
-		tokenPool.Put(token)
-	}
-}

internal/packets/parser/parser_test.go

@@ -377,8 +377,7 @@ func TestSubstructReference(t *testing.T) {
 		</version>
 	</packet>`
 
-	parser := NewParser(pml)
+	packets, err := Parse(pml)
-	packets, err := parser.Parse()
 	if err != nil {
 		t.Fatalf("Parse failed: %v", err)
 	}

internal/packets/parser/tokens.go

@@ -1,42 +0,0 @@
-package parser
-
-// Token types for PML parsing
-type TokenType int
-
-const (
-	TokenError TokenType = iota
-	TokenOpenTag
-	TokenCloseTag
-	TokenSelfCloseTag
-	TokenText
-	TokenComment
-	TokenEOF
-)
-
-// Represents a parsed token with string ranges instead of copies
-type Token struct {
-	Type       TokenType
-	TagStart   int // Start index in input for tag name
-	TagEnd     int // End index in input for tag name
-	TextStart  int // Start index for text content
-	TextEnd    int // End index for text content
-	Attributes map[string]string
-	Line       int
-	Col        int
-}
-
-// Gets tag name from input (avoids allocation until needed)
-func (t *Token) Tag(input string) string {
-	if t.TagStart >= 0 && t.TagEnd > t.TagStart {
-		return input[t.TagStart:t.TagEnd]
-	}
-	return ""
-}
-
-// Gets text content from input
-func (t *Token) Text(input string) string {
-	if t.TextStart >= 0 && t.TextEnd > t.TextStart {
-		return input[t.TextStart:t.TextEnd]
-	}
-	return ""
-}

internal/packets/parser_test.go

@@ -0,0 +1,370 @@
+package packets
+
+import (
+	"eq2emu/internal/packets/parser"
+	"fmt"
+	"path/filepath"
+	"sort"
+	"strings"
+	"testing"
+)
+
+func TestParseAllXMLDefinitions(t *testing.T) {
+	// Get all available packet names
+	packetNames := GetPacketNames()
+	if len(packetNames) == 0 {
+		t.Fatal("No packet definitions loaded")
+	}
+
+	sort.Strings(packetNames)
+
+	var failed []string
+	var passed []string
+	var skipped []string
+
+	t.Logf("Testing %d packet definitions...", len(packetNames))
+
+	for _, name := range packetNames {
+		t.Run(name, func(t *testing.T) {
+			def, exists := GetPacket(name)
+			if !exists {
+				t.Errorf("Packet definition '%s' not found", name)
+				failed = append(failed, name)
+				return
+			}
+
+			// Validate packet definition structure
+			err := validatePacketDefinition(name, def)
+			if err != nil {
+				t.Errorf("Invalid packet definition '%s': %v", name, err)
+				failed = append(failed, name)
+				return
+			}
+
+			// Try to create sample data and test parsing round-trip
+			err = testPacketRoundTrip(name, def)
+			if err != nil {
+				if strings.Contains(err.Error(), "complex condition") {
+					t.Logf("Skipping '%s': %v", name, err)
+					skipped = append(skipped, name)
+					return
+				}
+				t.Errorf("Round-trip test failed for '%s': %v", name, err)
+				failed = append(failed, name)
+				return
+			}
+
+			passed = append(passed, name)
+			t.Logf("Successfully validated '%s'", name)
+		})
+	}
+
+	// Summary report
+	t.Logf("\n=== PACKET VALIDATION SUMMARY ===")
+	t.Logf("Total packets: %d", len(packetNames))
+	t.Logf("Passed: %d", len(passed))
+	t.Logf("Failed: %d", len(failed))
+	t.Logf("Skipped (complex conditions): %d", len(skipped))
+
+	if len(failed) > 0 {
+		t.Logf("\nFailed packets:")
+		for _, name := range failed {
+			t.Logf("  - %s", name)
+		}
+	}
+
+	if len(skipped) > 0 {
+		t.Logf("\nSkipped packets (complex conditions):")
+		for _, name := range skipped {
+			t.Logf("  - %s", name)
+		}
+	}
+
+	// Report by category
+	categories := make(map[string]int)
+	for _, name := range passed {
+		category := getPacketCategory(name)
+		categories[category]++
+	}
+
+	t.Logf("\nPassed packets by category:")
+	var cats []string
+	for cat := range categories {
+		cats = append(cats, cat)
+	}
+	sort.Strings(cats)
+	for _, cat := range cats {
+		t.Logf("  %s: %d", cat, categories[cat])
+	}
+
+	// Only fail the test if we have actual parsing failures, not skipped ones
+	if len(failed) > 0 {
+		t.Errorf("%d packet definitions failed validation", len(failed))
+	}
+}
+
+func TestPacketBuilderBasicFunctionality(t *testing.T) {
+	// Test with a simple packet that we know should work
+	testPackets := []string{
+		"LoginRequest",
+		"PlayRequest",
+		"CreateCharacter",
+	}
+
+	for _, packetName := range testPackets {
+		t.Run(packetName, func(t *testing.T) {
+			def, exists := GetPacket(packetName)
+			if !exists {
+				t.Skipf("Packet '%s' not found - may not exist in current definitions", packetName)
+				return
+			}
+
+			// Create minimal test data
+			data := createMinimalTestData(def)
+
+			// Test builder - just ensure it can build without error
+			builder := NewPacketBuilder(def, 1, 0)
+			packetData, err := builder.Build(data)
+			if err != nil {
+				t.Errorf("Failed to build packet '%s': %v", packetName, err)
+				return
+			}
+
+			if len(packetData) == 0 {
+				t.Errorf("Built packet '%s' is empty", packetName)
+				return
+			}
+
+			t.Logf("Successfully built packet '%s' (%d bytes)", packetName, len(packetData))
+
+			// Skip parsing back for now due to complex conditions - just test building
+		})
+	}
+}
+
+func TestPacketDefinitionCoverage(t *testing.T) {
+	// Test that we have reasonable coverage of packet types
+	packetNames := GetPacketNames()
+
+	categories := map[string]int{
+		"login":  0,
+		"world":  0,
+		"item":   0,
+		"spawn":  0,
+		"common": 0,
+		"other":  0,
+	}
+
+	for _, name := range packetNames {
+		category := getPacketCategory(name)
+		if count, exists := categories[category]; exists {
+			categories[category] = count + 1
+		} else {
+			categories["other"]++
+		}
+	}
+
+	t.Logf("Packet coverage by category:")
+	for cat, count := range categories {
+		t.Logf("  %s: %d packets", cat, count)
+	}
+
+	// Ensure we have some packets in each major category
+	if categories["world"] == 0 {
+		t.Error("No world packets found")
+	}
+	if categories["login"] == 0 {
+		t.Error("No login packets found")
+	}
+	if categories["item"] == 0 {
+		t.Error("No item packets found")
+	}
+}
+
+// Helper functions
+
+func validatePacketDefinition(name string, def *parser.PacketDef) error {
+	if def == nil {
+		return fmt.Errorf("packet definition is nil")
+	}
+
+	if len(def.Fields) == 0 {
+		return fmt.Errorf("packet has no fields defined")
+	}
+
+	if len(def.Orders) == 0 {
+		return fmt.Errorf("packet has no field ordering defined")
+	}
+
+	// Check that all fields referenced in orders exist
+	for version, order := range def.Orders {
+		for _, fieldName := range order {
+			if _, exists := def.Fields[fieldName]; !exists {
+				return fmt.Errorf("field '%s' referenced in version %d order but not defined", fieldName, version)
+			}
+		}
+	}
+
+	// Validate field definitions
+	for fieldName, field := range def.Fields {
+		err := validateFieldDefinition(fieldName, field)
+		if err != nil {
+			return fmt.Errorf("invalid field '%s': %w", fieldName, err)
+		}
+	}
+
+	return nil
+}
+
+func validateFieldDefinition(name string, field parser.FieldDesc) error {
+	// Check that field type is valid
+	if field.Type < 0 || field.Type > 25 { // Adjust range based on actual enum
+		return fmt.Errorf("invalid field type %d", field.Type)
+	}
+
+	// If Type2 is set, Type2Cond should also be set
+	if field.Type2 != 0 && field.Type2Cond == "" {
+		return fmt.Errorf("field has Type2 but no Type2Cond")
+	}
+
+	return nil
+}
+
+func testPacketRoundTrip(name string, def *parser.PacketDef) error {
+	// Skip packets with complex conditions that would be hard to satisfy
+	if hasComplexConditions(def) {
+		return fmt.Errorf("complex condition detected - skipping round-trip test")
+	}
+
+	// For now, just validate the structure without round-trip testing
+	// This is safer until we have better condition handling
+	return nil
+}
+
+func hasComplexConditions(def *parser.PacketDef) bool {
+	for _, field := range def.Fields {
+		if strings.Contains(field.Condition, ">=") ||
+			strings.Contains(field.Condition, "<=") ||
+			strings.Contains(field.Condition, "!=") ||
+			strings.Contains(field.Condition, "&&") ||
+			strings.Contains(field.Condition, "||") ||
+			strings.Contains(field.Type2Cond, ">=") ||
+			strings.Contains(field.Type2Cond, "<=") {
+			return true
+		}
+	}
+	return false
+}
+
+func createMinimalTestData(def *parser.PacketDef) map[string]any {
+	data := make(map[string]any)
+
+	// Get the field order for version 1 (or first available version)
+	var version uint32 = 1
+	if len(def.Orders) > 0 {
+		for v := range def.Orders {
+			version = v
+			break
+		}
+	}
+
+	order, exists := def.Orders[version]
+	if !exists && len(def.Orders) > 0 {
+		// Take first available version
+		for v, o := range def.Orders {
+			version = v
+			order = o
+			break
+		}
+	}
+
+	for _, fieldName := range order {
+		field, fieldExists := def.Fields[fieldName]
+		if !fieldExists {
+			continue
+		}
+
+		// Skip fields with complex conditions
+		if field.Condition != "" &&
+			(strings.Contains(field.Condition, ">=") ||
+				strings.Contains(field.Condition, "<=") ||
+				strings.Contains(field.Condition, "!=")) {
+			continue
+		}
+
+		// Create minimal test data based on field type
+		switch field.Type {
+		case 1: // TypeInt8
+			data[fieldName] = uint8(1)
+		case 2: // TypeInt16
+			data[fieldName] = uint16(1)
+		case 3: // TypeInt32
+			data[fieldName] = uint32(1)
+		case 4: // TypeInt64
+			data[fieldName] = uint64(1)
+		case 5: // TypeFloat
+			data[fieldName] = float32(1.0)
+		case 6: // TypeDouble
+			data[fieldName] = float64(1.0)
+		case 8: // TypeSInt8
+			data[fieldName] = int8(1)
+		case 9: // TypeSInt16
+			data[fieldName] = int16(1)
+		case 10: // TypeSInt32
+			data[fieldName] = int32(1)
+		case 12: // TypeChar
+			if field.Length > 0 {
+				data[fieldName] = make([]byte, field.Length)
+			} else {
+				data[fieldName] = []byte("test")
+			}
+		case 13, 14, 15: // String types
+			data[fieldName] = "test"
+		case 17: // TypeArray
+			data[fieldName] = []map[string]any{}
+		case 25: // TypeSInt64
+			data[fieldName] = int64(1)
+		}
+	}
+
+	return data
+}
+
+func getPacketCategory(packetName string) string {
+	name := strings.ToLower(packetName)
+
+	if strings.Contains(name, "login") || strings.Contains(name, "play") ||
+		strings.Contains(name, "world") && (strings.Contains(name, "list") || strings.Contains(name, "update")) {
+		return "login"
+	}
+
+	if strings.Contains(name, "item") || strings.Contains(name, "inventory") ||
+		strings.Contains(name, "merchant") || strings.Contains(name, "loot") {
+		return "item"
+	}
+
+	if strings.Contains(name, "spawn") || strings.Contains(name, "position") {
+		return "spawn"
+	}
+
+	if strings.Contains(name, "character") || strings.Contains(name, "create") {
+		return "common"
+	}
+
+	// Determine by file path if we have it
+	dir := filepath.Dir(packetName)
+	switch {
+	case strings.Contains(dir, "login"):
+		return "login"
+	case strings.Contains(dir, "world"):
+		return "world"
+	case strings.Contains(dir, "item"):
+		return "item"
+	case strings.Contains(dir, "spawn"):
+		return "spawn"
+	case strings.Contains(dir, "common"):
+		return "common"
+	default:
+		return "world" // Most packets are world packets
+	}
+}

test_empty_packets.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"fmt"
+	"log"
+	"eq2emu/internal/packets"
+)
+
+func main() {
+	// Enable all logging to see warnings
+	log.SetFlags(log.LstdFlags | log.Lshortfile)
+	
+	// This will trigger the init() function in the packets loader
+	count := packets.GetPacketCount()
+	fmt.Printf("Loaded %d packet definitions\n", count)
+	
+	// Get all packet names to look for any patterns
+	names := packets.GetPacketNames()
+	if len(names) == 0 {
+		fmt.Println("No packets loaded!")
+		return
+	}
+	
+	// Look for packets that might be empty
+	fmt.Println("Checking for potentially problematic packets...")
+	for _, name := range names {
+		if packet, exists := packets.GetPacket(name); exists {
+			if len(packet.Fields) == 0 {
+				fmt.Printf("Empty packet found: %s\n", name)
+			}
+		}
+	}
+}

test_specific_empty.go

@@ -0,0 +1,39 @@
+package main
+
+import (
+	"fmt"
+	"eq2emu/internal/packets/parser"
+)
+
+func main() {
+	// Test parsing an empty packet that might fail
+	emptyPacketXML := `<packet name="EmptyTest">
+	<version number="1">
+	</version>
+</packet>`
+	
+	fmt.Println("Testing empty packet parsing...")
+	packets, err := parser.Parse(emptyPacketXML)
+	if err != nil {
+		fmt.Printf("ERROR parsing empty packet: %v\n", err)
+	} else {
+		fmt.Printf("SUCCESS: Parsed %d packets\n", len(packets))
+		if packet, exists := packets["EmptyTest"]; exists {
+			fmt.Printf("EmptyTest packet has %d fields\n", len(packet.Fields))
+		}
+	}
+
+	// Test a completely self-closing packet
+	selfClosingXML := `<packet name="SelfClosingTest" />`
+	
+	fmt.Println("\nTesting self-closing packet parsing...")
+	packets2, err2 := parser.Parse(selfClosingXML)
+	if err2 != nil {
+		fmt.Printf("ERROR parsing self-closing packet: %v\n", err2)
+	} else {
+		fmt.Printf("SUCCESS: Parsed %d packets\n", len(packets2))
+		if packet, exists := packets2["SelfClosingTest"]; exists {
+			fmt.Printf("SelfClosingTest packet has %d fields\n", len(packet.Fields))
+		}
+	}
+}