Compare commits

..

No commits in common. "fb359223f0ba3ca14c4e83689891507cce00cb91" and "b537916c6b0367d2f8274711545c2c86f38a376d" have entirely different histories.

5 changed files with 96 additions and 207 deletions

View File

@ -20,8 +20,7 @@ add_subdirectory(external/raylib-cpp-4.5.1)
add_subdirectory(external/utfcpp-3.2.5) add_subdirectory(external/utfcpp-3.2.5)
add_executable(browser src/main.cpp src/fetch.cpp add_executable(browser src/main.cpp src/fetch.cpp
src/html_parse.cpp src/html_parse.cpp)
src/utf8.hpp)
target_include_directories(browser PRIVATE external/raygui-4.0/src) target_include_directories(browser PRIVATE external/raygui-4.0/src)

View File

@ -1,8 +1,8 @@
#include "html_parse.hpp" #include "html_parse.hpp"
#include <cassert> #include <cassert>
#include <charconv>
#include <iostream> #include <iostream>
#include <utility>
#include <utf8.h> #include <utf8.h>
@ -40,6 +40,7 @@ static bool is_newline(char32_t c)
std::vector<Token> Tokenizer::tokenize() std::vector<Token> Tokenizer::tokenize()
{ {
std::string buffer;
std::vector<Token> tokens; std::vector<Token> tokens;
utf8::iterator<std::string::iterator> it(m_source.begin(), m_source.begin(), m_source.end()); utf8::iterator<std::string::iterator> it(m_source.begin(), m_source.begin(), m_source.end());
@ -47,84 +48,91 @@ std::vector<Token> Tokenizer::tokenize()
for (; it != end; ++it) { for (; it != end; ++it) {
if (is_alpha(*it)) { if (is_alpha(*it)) {
auto begin = it; do {
while (++it != end && (is_alpha_num(*it) || *it == '-')) { } utf8::append(*it, std::back_inserter(buffer));
tokens.push_back({ .type = TokenType::ident, .view { begin, it } }); } while (++it != end && (is_alpha_num(*it) || *it == '-'));
--it; --it;
tokens.push_back({ .type = TokenType::ident, .value = buffer });
buffer.clear();
} }
else if (*it == '<') { else if (*it == '<') {
tokens.push_back({ .type = TokenType::lt, .view { it } }); tokens.push_back({ .type = TokenType::lt });
} }
else if (*it == '>') { else if (*it == '>') {
tokens.push_back({ .type = TokenType::gt, .view { it } }); tokens.push_back({ .type = TokenType::gt });
} }
else if (*it == '!') { else if (*it == '!') {
tokens.push_back({ .type = TokenType::exclaim, .view { it } }); tokens.push_back({ .type = TokenType::exclaim });
} }
else if (*it == '/') { else if (*it == '/') {
tokens.push_back({ .type = TokenType::fslash, .view { it } }); tokens.push_back({ .type = TokenType::fslash });
} }
else if (*it == '=') { else if (*it == '=') {
tokens.push_back({ .type = TokenType::eq, .view { it } }); tokens.push_back({ .type = TokenType::eq });
} }
else if (*it == '-') { else if (*it == '-') {
tokens.push_back({ .type = TokenType::minus, .view { it } }); tokens.push_back({ .type = TokenType::minus });
} }
else if (*it == ':') { else if (*it == ':') {
tokens.push_back({ .type = TokenType::colon, .view { it } }); tokens.push_back({ .type = TokenType::colon });
} }
else if (*it == '#') { else if (*it == '#') {
tokens.push_back({ .type = TokenType::hash, .view { it } }); tokens.push_back({ .type = TokenType::hash });
} }
else if (*it == ';') { else if (*it == ';') {
tokens.push_back({ .type = TokenType::semi, .view { it } }); tokens.push_back({ .type = TokenType::semi });
} }
else if (is_num(*it)) { else if (is_num(*it)) {
auto begin = it; do {
while (++it != end && is_num(*it)) { } utf8::append(*it, std::back_inserter(buffer));
tokens.push_back({ .type = TokenType::num, .view { begin, it } }); } while (++it != end && is_num(*it));
--it; --it;
tokens.push_back({ .type = TokenType::num, .value = buffer });
buffer.clear();
} }
else if (*it == ',') { else if (*it == ',') {
tokens.push_back({ .type = TokenType::comma, .view { it } }); tokens.push_back({ .type = TokenType::comma });
} }
else if (*it == '{') { else if (*it == '{') {
tokens.push_back({ .type = TokenType::left_curly, .view { it } }); tokens.push_back({ .type = TokenType::left_curly });
} }
else if (*it == '}') { else if (*it == '}') {
tokens.push_back({ .type = TokenType::right_curly, .view { it } }); tokens.push_back({ .type = TokenType::right_curly });
} }
else if (*it == '(') { else if (*it == '(') {
tokens.push_back({ .type = TokenType::left_paren, .view { it } }); tokens.push_back({ .type = TokenType::left_paren });
} }
else if (*it == ')') { else if (*it == ')') {
tokens.push_back({ .type = TokenType::right_paren, .view { it } }); tokens.push_back({ .type = TokenType::right_paren });
} }
else if (*it == '.') { else if (*it == '.') {
tokens.push_back({ .type = TokenType::dot, .view { it } }); tokens.push_back({ .type = TokenType::dot });
} }
else if (*it == '@') { else if (*it == '@') {
tokens.push_back({ .type = TokenType::at, .view { it } }); tokens.push_back({ .type = TokenType::at });
} }
else if (*it == '"') { else if (*it == '"') {
auto begin = it;
++it; ++it;
while (it != end && *it != '"') { while (it != end && *it != '"') {
utf8::append(*it, std::back_inserter(buffer));
++it; ++it;
} }
tokens.push_back({ .type = TokenType::str, .view { begin, it } }); tokens.push_back({ .type = TokenType::str, .value = buffer });
buffer.clear();
} }
else if (is_space(*it) || is_newline(*it)) { else if (is_space(*it) || is_newline(*it)) {
} }
else { else {
std::cout << "[WARN] Unexpected token" << std::endl; utf8::append(*it, std::back_inserter(buffer));
std::cout << "[WARN] Unexpected token: " << buffer << std::endl;
buffer.clear();
} }
} }
return tokens; return tokens;
} }
std::string Token::to_string() const std::string Token::to_string()
{ {
switch (type) { switch (type) {
case TokenType::lt: case TokenType::lt:
@ -132,11 +140,13 @@ std::string Token::to_string() const
case TokenType::gt: case TokenType::gt:
return ">"; return ">";
case TokenType::ident: case TokenType::ident:
return "IDENT"; // TODO assert(value.has_value());
return value.value();
case TokenType::fslash: case TokenType::fslash:
return "/"; return "/";
case TokenType::str: case TokenType::str:
return "STR"; // TODO assert(value.has_value());
return value.value();
case TokenType::exclaim: case TokenType::exclaim:
return "!"; return "!";
case TokenType::eq: case TokenType::eq:
@ -164,7 +174,8 @@ std::string Token::to_string() const
case TokenType::semi: case TokenType::semi:
return ";"; return ";";
case TokenType::num: case TokenType::num:
return "NUM"; // TODO assert(value.has_value());
return value.value();
default: default:
assert(false && "Unimplemented"); assert(false && "Unimplemented");
return ""; return "";
@ -185,10 +196,10 @@ NodeDoc Parser::parse()
} }
while (peek().has_value()) { while (peek().has_value()) {
if (auto elem = parse_elem()) { if (auto elem = parse_elem()) {
doc.children.emplace_back(elem.value()); doc.children.push_back(elem.value());
} }
else { else {
doc.children.emplace_back(Utf8String(consume().view)); doc.children.push_back(consume().to_string());
} }
} }
return doc; return doc;
@ -222,12 +233,13 @@ bool is_ci_equal(const std::string& s1, const std::string& s2)
std::optional<NodeDocType> Parser::parse_doc_type() std::optional<NodeDocType> Parser::parse_doc_type()
{ {
if (peek_is(0, TokenType::lt) && peek_is(1, TokenType::exclaim) if (peek_is(0, TokenType::lt) && peek_is(1, TokenType::exclaim)
&& peek_is(2, TokenType::ident, Utf8View("doctype"), StrCmp::case_insensitive) && peek_is(3, TokenType::ident) && peek_is(2, TokenType::ident, "doctype", StrCmp::case_insensitive) && peek_is_with_val(3, TokenType::ident)
&& peek_is(4, TokenType::gt)) { && peek_is(4, TokenType::gt)) {
consume(); consume();
consume(); consume();
consume(); consume();
NodeDocType doc_type { .type = Utf8String(consume().view) }; NodeDocType doc_type;
doc_type.type = consume().value.value();
consume(); consume();
return doc_type; return doc_type;
} }
@ -242,40 +254,49 @@ bool Parser::peek_is(size_t ahead, TokenType type)
return peek(ahead).value().get().type == type; return peek(ahead).value().get().type == type;
} }
bool Parser::peek_is(size_t ahead, TokenType type, const Utf8View& val, Parser::StrCmp cmp) bool Parser::peek_is_with_val(size_t ahead, TokenType type)
{ {
if (!peek_is(ahead, type)) { if (!peek_is(ahead, type)) {
return false; return false;
} }
switch (cmp) { return peek(ahead).value().get().value.has_value();
case StrCmp::case_sensitive:
return peek(ahead).value().get().view == val;
case StrCmp::case_insensitive:
return peek(ahead).value().get().view.case_ins_equals(val);
}
} }
bool Parser::peek_is(size_t ahead, TokenType type, const std::string& val, Parser::StrCmp cmp)
{
if (!peek_is_with_val(ahead, type)) {
return false;
}
switch (cmp) {
case StrCmp::case_sensitive:
return peek(ahead).value().get().value.value() == val;
case StrCmp::case_insensitive:
return is_ci_equal(peek(ahead).value().get().value.value(), val);
}
}
std::optional<NodeElem> Parser::parse_elem() std::optional<NodeElem> Parser::parse_elem()
{ {
if (peek_is(0, TokenType::lt) && peek_is(1, TokenType::ident)) { if (peek_is(0, TokenType::lt) && peek_is_with_val(1, TokenType::ident)) {
consume(); consume();
Utf8View tag = consume().view; std::string tag = consume().value.value();
std::vector<NodeAttr> attributes; std::vector<NodeAttr> attributes;
while (auto attr = parse_attr()) { while (auto attr = parse_attr()) {
attributes.push_back(attr.value()); attributes.push_back(attr.value());
} }
if (peek_is(0, TokenType::gt)) { if (peek_is(0, TokenType::gt)) {
consume(); consume();
NodeElemReg elem_reg { .tag = Utf8String(tag), .attributes = std::move(attributes) }; NodeElemReg elem_reg;
elem_reg.tag = tag;
elem_reg.attributes = std::move(attributes);
while (peek().has_value() while (peek().has_value()
&& !( && !(
peek_is(0, TokenType::lt) && peek_is(1, TokenType::fslash) peek_is(0, TokenType::lt) && peek_is(1, TokenType::fslash)
&& peek_is(2, TokenType::ident, tag, StrCmp::case_insensitive) && peek_is(3, TokenType::gt))) { && peek_is(2, TokenType::ident, tag, StrCmp::case_insensitive) && peek_is(3, TokenType::gt))) {
if (auto child = parse_elem()) { if (auto child = parse_elem()) {
elem_reg.inner.emplace_back(child.value()); elem_reg.inner.push_back(child.value());
} }
else if (peek().has_value()) { else if (peek().has_value()) {
elem_reg.inner.emplace_back(Utf8String(consume().view)); elem_reg.inner.push_back(consume().to_string());
} }
else { else {
assert(false && "Unexpected"); assert(false && "Unexpected");
@ -287,7 +308,8 @@ std::optional<NodeElem> Parser::parse_elem()
consume(); consume();
consume(); consume();
consume(); consume();
NodeElem elem { .var = std::move(elem_reg) }; NodeElem elem;
elem.var = std::move(elem_reg);
return elem; return elem;
} }
else { else {
@ -295,13 +317,16 @@ std::optional<NodeElem> Parser::parse_elem()
} }
} }
else if (peek_is(0, TokenType::fslash)) { else if (peek_is(0, TokenType::fslash)) {
NodeElemSelfClose elem_close { .tag = Utf8String(tag), .attributes = std::move(attributes) }; NodeElemSelfClose elem_close;
elem_close.tag = tag;
elem_close.attributes = std::move(attributes);
consume(); consume();
if (!peek_is(0, TokenType::gt)) { if (!peek_is(0, TokenType::gt)) {
assert(false && "Unexpected"); assert(false && "Unexpected");
} }
consume(); consume();
NodeElem elem { .var = std::move(elem_close) }; NodeElem elem;
elem.var = std::move(elem_close);
return elem; return elem;
} }
else { else {
@ -312,12 +337,12 @@ std::optional<NodeElem> Parser::parse_elem()
} }
std::optional<NodeAttr> Parser::parse_attr() std::optional<NodeAttr> Parser::parse_attr()
{ {
if (peek_is(0, TokenType::ident) && peek_is(1, TokenType::eq) if (peek_is_with_val(0, TokenType::ident) && peek_is(1, TokenType::eq)
&& (peek_is(2, TokenType::ident) || peek_is(2, TokenType::str))) { && (peek_is_with_val(2, TokenType::ident) || peek_is_with_val(2, TokenType::str))) {
Utf8String key = Utf8String(consume().view); NodeAttr attr;
attr.key = consume().value.value();
consume(); consume();
Utf8String val = Utf8String(consume().view); attr.val = consume().value.value();
NodeAttr attr { .key = key, .val = val };
return attr; return attr;
} }
else { else {
@ -325,4 +350,4 @@ std::optional<NodeAttr> Parser::parse_attr()
} }
} }
} }

View File

@ -6,12 +6,11 @@
#include <variant> #include <variant>
#include <vector> #include <vector>
#include "utf8.hpp" #include <utf8.h>
namespace html { namespace html {
enum class TokenType { enum class TokenType {
unknown,
lt, lt,
gt, gt,
ident, ident,
@ -34,10 +33,10 @@ enum class TokenType {
}; };
struct Token { struct Token {
TokenType type = TokenType::unknown; TokenType type;
Utf8View view; std::optional<std::string> value = {};
std::string to_string() const; std::string to_string();
}; };
class Tokenizer { class Tokenizer {
@ -51,24 +50,24 @@ private:
}; };
struct NodeDocType { struct NodeDocType {
Utf8String type; std::string type;
}; };
struct NodeAttr { struct NodeAttr {
Utf8String key; std::string key;
Utf8String val; std::string val;
}; };
struct NodeElem; struct NodeElem;
struct NodeElemReg { struct NodeElemReg {
Utf8String tag; std::string tag;
std::vector<NodeAttr> attributes; std::vector<NodeAttr> attributes;
std::vector<std::variant<NodeElem, Utf8String>> inner; std::vector<std::variant<NodeElem, std::string>> inner;
}; };
struct NodeElemSelfClose { struct NodeElemSelfClose {
Utf8String tag; std::string tag;
std::vector<NodeAttr> attributes; std::vector<NodeAttr> attributes;
}; };
@ -78,7 +77,7 @@ struct NodeElem {
struct NodeDoc { struct NodeDoc {
std::optional<NodeDocType> doc_type {}; std::optional<NodeDocType> doc_type {};
std::vector<std::variant<NodeElem, Utf8String>> children {}; std::vector<std::variant<NodeElem, std::string>> children {};
}; };
class Parser { class Parser {
@ -97,7 +96,8 @@ private:
enum class StrCmp { case_sensitive, case_insensitive }; enum class StrCmp { case_sensitive, case_insensitive };
bool peek_is(size_t ahead, TokenType type); bool peek_is(size_t ahead, TokenType type);
bool peek_is(size_t ahead, TokenType type, const Utf8View& val, StrCmp cmp); bool peek_is_with_val(size_t ahead, TokenType type);
bool peek_is(size_t ahead, TokenType type, const std::string& val, StrCmp cmp);
Token& consume(); Token& consume();

View File

@ -13,8 +13,7 @@ int main()
{ {
init_curl(); init_curl();
// std::optional<std::string> page_data = fetch_url("test://text.html"); std::optional<std::string> page_data = fetch_url("test://text.html");
std::optional<std::string> page_data = fetch_url("https://example.com");
SetConfigFlags(ConfigFlags::FLAG_WINDOW_RESIZABLE | ConfigFlags::FLAG_MSAA_4X_HINT | ConfigFlags ::FLAG_VSYNC_HINT); SetConfigFlags(ConfigFlags::FLAG_WINDOW_RESIZABLE | ConfigFlags::FLAG_MSAA_4X_HINT | ConfigFlags ::FLAG_VSYNC_HINT);
@ -27,8 +26,7 @@ int main()
SetTargetFPS(60); SetTargetFPS(60);
bool is_editing_url = false; bool is_editing_url = false;
std::string url_input = "https://example.com"; std::string url_input = "test://text.html";
// std::string url_input = "test://text.html";
url_input.reserve(1024); url_input.reserve(1024);
float scroll_pos = 0.0f; float scroll_pos = 0.0f;

View File

@ -1,133 +0,0 @@
#include <utf8.h>
inline uint32_t utf8_to_lower(uint32_t c)
{
if (c >= 65 && c <= 90) {
return c + 32;
}
return c;
}
class Utf8View {
public:
using Iterator = utf8::iterator<std::string::iterator>;
using ConstIterator = utf8::iterator<std::string::const_iterator>;
inline explicit Utf8View(const std::string& str)
: m_begin(str.begin())
, m_end(str.end())
{
}
inline explicit Utf8View(const Iterator& begin)
: m_begin(begin.base())
, m_end(begin.base() + 1)
{
}
inline Utf8View(const Iterator& begin, const Iterator& end)
: m_begin(begin.base())
, m_end(end.base())
{
}
[[nodiscard]] inline bool operator==(const Utf8View& other) const
{
if (size() != other.size()) {
return false;
}
for (int64_t i = 0; i < size(); i++) {
if ((*this)[i] != other[i]) {
return false;
}
}
return true;
}
[[nodiscard]] inline bool case_ins_equals(const Utf8View& other) const
{
if (size() != other.size()) {
return false;
}
for (int64_t i = 0; i < size(); i++) {
if (utf8_to_lower((*this)[i]) != utf8_to_lower(other[i])) {
return false;
}
}
return true;
}
[[nodiscard]] inline size_t size() const
{
return m_end - m_begin;
}
[[nodiscard]] inline uint32_t operator[](int64_t index) const
{
return *(m_begin + index);
}
[[nodiscard]] inline ConstIterator cbegin() const
{
return ConstIterator(m_begin, m_begin, m_end);
}
[[nodiscard]] inline ConstIterator cend() const
{
return ConstIterator(m_end, m_begin, m_end);
}
private:
std::string::const_iterator m_begin;
std::string::const_iterator m_end;
};
class Utf8String {
public:
using Iterator = utf8::iterator<std::string::iterator>;
using ConstIterator = utf8::iterator<std::string::const_iterator>;
explicit inline Utf8String(std::string str)
: m_str(std::move(str))
{
}
explicit inline Utf8String(const Utf8View& view)
: m_str(view.cbegin().base(), view.cend().base())
{
}
inline Iterator begin()
{
return Iterator(m_str.begin(), m_str.begin(), m_str.end());
}
inline Iterator end()
{
return Iterator(m_str.end(), m_str.begin(), m_str.end());
}
[[nodiscard]] inline ConstIterator cbegin() const
{
return ConstIterator(m_str.begin(), m_str.begin(), m_str.end());
}
[[nodiscard]] inline ConstIterator cend() const
{
return ConstIterator(m_str.end(), m_str.begin(), m_str.end());
}
bool operator==(const Utf8String& other) const
{
return m_str == other.m_str;
}
uint32_t operator[](size_t index)
{
auto it = begin();
for (size_t i = 0; i < index; i++) {
++it;
}
return *it;
}
private:
std::string m_str;
};