Initial tokenization

This commit is contained in:
orosmatthew 2023-09-28 11:15:49 -04:00
parent 5ffeafb4de
commit 474d4305f7
4 changed files with 101 additions and 8 deletions

View File

@ -17,7 +17,8 @@ add_subdirectory(external/raylib-4.5.0)
add_subdirectory(external/raylib-cpp-4.5.1) add_subdirectory(external/raylib-cpp-4.5.1)
add_executable(browser src/main.cpp src/fetch.cpp) add_executable(browser src/main.cpp src/fetch.cpp
src/html_parse.cpp)
target_include_directories(browser PRIVATE external/raygui-4.0/src) target_include_directories(browser PRIVATE external/raygui-4.0/src)

46
src/html_parse.cpp Normal file
View File

@ -0,0 +1,46 @@
#include "html_parse.hpp"
#include <iostream>
namespace html {
Tokenizer::Tokenizer(std::u32string source)
: m_source(std::move(source))
, m_index(0)
{
}
std::vector<Token> Tokenizer::tokenize()
{
std::vector<Token> tokens;
while (peek().has_value()) {
if (peek().value() == '<') {
consume();
tokens.push_back({ .type = TokenType::lt });
}
else if (peek().value() == '>') {
consume();
tokens.push_back({ .type = TokenType::gt });
}
else {
std::cout << "[WARN] Unexpected token: " << std::to_string(peek().value()) << std::endl;
consume();
}
}
return {};
}
std::optional<char32_t> Tokenizer::peek(size_t ahead)
{
if (m_index + ahead >= m_source.size()) {
return {};
}
return m_source.at(m_index + ahead);
}
char32_t Tokenizer::consume()
{
return m_source.at(m_index++);
}
}

43
src/html_parse.hpp Normal file
View File

@ -0,0 +1,43 @@
#pragma once
#include <optional>
#include <string>
#include <vector>
namespace html {
enum class TokenType {
lt,
gt,
ident,
fslash,
quote,
};
struct Token {
TokenType type;
std::optional<std::string> value = {};
};
enum class NodeType {
};
struct Node { };
class Tokenizer {
public:
explicit Tokenizer(std::u32string source);
std::vector<Token> tokenize();
private:
std::optional<char32_t> peek(size_t ahead = 1);
char32_t consume();
std::u32string m_source;
size_t m_index;
};
}

View File

@ -1,8 +1,9 @@
#include "fetch.hpp" #include "fetch.hpp"
#include <iostream>
#include <optional> #include <optional>
#include "html_parse.hpp"
#define RAYGUI_IMPLEMENTATION #define RAYGUI_IMPLEMENTATION
#include <raygui.h> #include <raygui.h>
#include <raylib-cpp.hpp> #include <raylib-cpp.hpp>
@ -11,7 +12,7 @@ int main()
{ {
init_curl(); init_curl();
std::optional<std::string> page_data; std::optional<std::string> page_data = fetch_url("https://example.com");
SetConfigFlags(ConfigFlags::FLAG_WINDOW_RESIZABLE | ConfigFlags::FLAG_MSAA_4X_HINT | ConfigFlags ::FLAG_VSYNC_HINT); SetConfigFlags(ConfigFlags::FLAG_WINDOW_RESIZABLE | ConfigFlags::FLAG_MSAA_4X_HINT | ConfigFlags ::FLAG_VSYNC_HINT);
@ -24,11 +25,16 @@ int main()
SetTargetFPS(60); SetTargetFPS(60);
bool is_editing_url = false; bool is_editing_url = false;
std::string url_input; std::string url_input = "https://example.com";
url_input.reserve(1024); url_input.reserve(1024);
raylib::Vector2 page_content_size;
float scroll_pos = 0.0f; float scroll_pos = 0.0f;
if (page_data.has_value()) {
std::u32string page_data_unicode(page_data.value().begin(), page_data.value().end());
html::Tokenizer tokenizer(page_data_unicode);
tokenizer.tokenize();
}
while (!window.ShouldClose()) { while (!window.ShouldClose()) {
BeginDrawing(); BeginDrawing();
window.ClearBackground(raylib::Color::White()); window.ClearBackground(raylib::Color::White());
@ -43,9 +49,6 @@ int main()
if (GuiButton({ .x = (float)GetScreenWidth() - 50, .y = 0, .width = 50, .height = 24 }, "Go")) { if (GuiButton({ .x = (float)GetScreenWidth() - 50, .y = 0, .width = 50, .height = 24 }, "Go")) {
page_data = fetch_url(url_input); page_data = fetch_url(url_input);
if (page_data.has_value()) {
page_content_size = MeasureTextEx(sans_font, page_data.value().data(), 24, 1.0f);
}
} }
scroll_pos += GetMouseWheelMove(); scroll_pos += GetMouseWheelMove();