fix leak

include/pocketpy/compiler/lexer.h

@@ -168,6 +168,7 @@ enum Precedence {
 };
 
 Error* Lexer__process(SourceData_ src, Token** out_tokens, int* out_length);
+void destruct_tokens(Token* tokens, int length);
 
 #define Token__sv(self)                                                                            \
     (c11_sv) { (self)->start, (self)->length }

src/compiler/compiler.c

@@ -1399,12 +1399,7 @@ static void Compiler__ctor(Compiler* self, SourceData_ src, Token* tokens, int t
 
 static void Compiler__dtor(Compiler* self) {
     // free tokens
-    for(int i = 0; i < self->tokens_length; i++) {
-        if(self->tokens[i].value.index == TokenValue_STR) {
-            // PK_FREE internal string
-            c11_string__delete(self->tokens[i].value._str);
-        }
-    }
+    destruct_tokens(self->tokens, self->tokens_length);
     PK_FREE(self->tokens);
     // free contexts
     c11__foreach(Ctx, &self->contexts, ctx) Ctx__dtor(ctx);

src/compiler/lexer.c

@@ -628,6 +628,7 @@ Error* Lexer__process(SourceData_ src, Token** out_tokens, int* out_length) {
     while(!eof) {
         void* err = lex_one_token(&lexer, &eof, false);
         if(err) {
+            destruct_tokens(lexer.nexts.data, lexer.nexts.length);
             Lexer__dtor(&lexer);
             return err;
         }
@@ -639,6 +640,16 @@ Error* Lexer__process(SourceData_ src, Token** out_tokens, int* out_length) {
     return NULL;
 }
 
+void destruct_tokens(Token* tokens, int length) {
+    // free tokens
+    for(int i = 0; i < length; i++) {
+        if(tokens[i].value.index == TokenValue_STR) {
+            // PK_FREE internal string
+            c11_string__delete(tokens[i].value._str);
+        }
+    }
+}
+
 const char* TokenSymbols[] = {
     "@eof",
     "@eol",