Always serialize the full depth of the tag stack

This commit is contained in:
Max Brunsfeld 2018-08-29 11:01:00 -07:00
parent 37b1bd4eef
commit 0f2d7e1083
2 changed files with 28 additions and 13 deletions

View File

@ -26,14 +26,18 @@ struct Scanner {
Scanner() {} Scanner() {}
unsigned serialize(char *buffer) { unsigned serialize(char *buffer) {
unsigned tag_count = 0; uint16_t tag_count = tags.size() > UINT16_MAX ? UINT16_MAX : tags.size();
unsigned i = sizeof(tag_count); uint16_t serialized_tag_count = 0;
for (unsigned n = tags.size(); tag_count < n; tag_count++) { unsigned i = sizeof(tag_count);
Tag &tag = tags[tag_count]; std::memcpy(&buffer[i], &tag_count, sizeof(tag_count));
i += sizeof(tag_count);
for (; serialized_tag_count < tag_count; serialized_tag_count++) {
Tag &tag = tags[serialized_tag_count];
if (tag.type == CUSTOM) { if (tag.type == CUSTOM) {
unsigned name_length = tag.custom_tag_name.size(); unsigned name_length = tag.custom_tag_name.size();
if (name_length > UINT8_MAX) break; if (name_length > UINT8_MAX) name_length = UINT8_MAX;
if (i + 2 + name_length >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) break; if (i + 2 + name_length >= TREE_SITTER_SERIALIZATION_BUFFER_SIZE) break;
buffer[i++] = static_cast<char>(tag.type); buffer[i++] = static_cast<char>(tag.type);
buffer[i++] = name_length; buffer[i++] = name_length;
@ -45,7 +49,7 @@ struct Scanner {
} }
} }
std::memcpy(buffer, &tag_count, sizeof(tag_count)); std::memcpy(&buffer[0], &serialized_tag_count, sizeof(serialized_tag_count));
return i; return i;
} }
@ -53,15 +57,20 @@ struct Scanner {
tags.clear(); tags.clear();
if (length > 0) { if (length > 0) {
unsigned i = 0; unsigned i = 0;
unsigned n; uint16_t tag_count, serialized_tag_count;
std::memcpy(&n, buffer, sizeof(n));
i += sizeof(n); std::memcpy(&serialized_tag_count, &buffer[i], sizeof(serialized_tag_count));
tags.resize(n); i += sizeof(serialized_tag_count);
for (unsigned j = 0; j < n; j++) {
std::memcpy(&tag_count, &buffer[i], sizeof(tag_count));
i += sizeof(tag_count);
tags.resize(tag_count);
for (unsigned j = 0; j < serialized_tag_count; j++) {
Tag &tag = tags[j]; Tag &tag = tags[j];
tag.type = static_cast<TagType>(buffer[i++]); tag.type = static_cast<TagType>(buffer[i++]);
if (tag.type == CUSTOM) { if (tag.type == CUSTOM) {
unsigned name_length = (unsigned char)buffer[i++]; uint16_t name_length = (uint16_t)buffer[i++];
tag.custom_tag_name.assign(&buffer[i], &buffer[i + name_length]); tag.custom_tag_name.assign(&buffer[i], &buffer[i + name_length]);
i += name_length; i += name_length;
} }

View File

@ -310,7 +310,13 @@ struct Tag {
TagType type; TagType type;
string custom_tag_name; string custom_tag_name;
Tag() : type(DIV) {} // This default constructor is used in the case where there is not enough space
// in the serialization buffer to store all of the tags. In that case, tags
// that cannot be serialized will be treated as having an unknown type. These
// tags will be closed via implicit end tags regardless of the next closing
// tag is encountered.
Tag() : type(END_OF_VOID_TAGS) {}
Tag(TagType type, const string &name) : type(type), custom_tag_name(name) {} Tag(TagType type, const string &name) : type(type), custom_tag_name(name) {}
bool operator==(const Tag &other) const { bool operator==(const Tag &other) const {