diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/epub.zig | 129 | ||||
| -rw-r--r-- | src/fetch.zig | 30 | ||||
| -rw-r--r-- | src/html.zig | 309 | ||||
| -rw-r--r-- | src/index.zig | 103 | ||||
| -rw-r--r-- | src/main.zig | 106 | ||||
| -rw-r--r-- | src/zip.zig | 192 |
6 files changed, 869 insertions, 0 deletions
diff --git a/src/epub.zig b/src/epub.zig new file mode 100644 index 0000000..0bdf563 --- /dev/null +++ b/src/epub.zig @@ -0,0 +1,129 @@ +//! Package transformed XHTML content into a valid EPUB3 file (a ZIP with a +//! specific structure). The `mimetype` entry must be first and stored uncompressed. + +const std = @import("std"); +const assert = std.debug.assert; +const zip = @import("zip.zig"); + +/// Fixed modification timestamp for reproducible output. +const modified = "2021-01-01T00:00:00Z"; + +pub const Options = struct { + title: []const u8, + /// Version name, used in the title and as the unique identifier basis. + version: []const u8, + lang: []const u8 = "en", + /// Well-formed XHTML5 body (output of html.toXhtml). + xhtml: []const u8, +}; + +/// Build an EPUB3 archive from `opts`. Caller owns the returned bytes. +pub fn build(allocator: std.mem.Allocator, opts: Options) ![]u8 { + assert(opts.version.len > 0); + assert(opts.xhtml.len > 0); + + var w = zip.Writer.init(allocator); + defer w.deinit(); + + // 1. mimetype — MUST be the first entry and stored uncompressed. + try w.addStored("mimetype", "application/epub+zip"); + + // 2. container.xml — points the reader at the package document. + try w.addStored("META-INF/container.xml", container_xml); + + // 3. content.opf — package metadata, manifest, spine. + const opf = try buildOpf(allocator, opts); + defer allocator.free(opf); + try w.addStored("OEBPS/content.opf", opf); + + // 4. nav.xhtml — minimal EPUB3 navigation document. + const nav = try buildNav(allocator, opts); + defer allocator.free(nav); + try w.addStored("OEBPS/nav.xhtml", nav); + + // 5. index.xhtml — the transformed reference content. + try w.addStored("OEBPS/index.xhtml", opts.xhtml); + + const bytes = try w.finish(); + assert(std.mem.startsWith(u8, bytes, "PK")); + return bytes; +} + +const container_xml = + \\<?xml version="1.0" encoding="utf-8"?> + \\<container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container"> + \\ <rootfiles> + \\ <rootfile full-path="OEBPS/content.opf" media-type="application/oebps-package+xml"/> + \\ </rootfiles> + \\</container> + \\ +; + +fn buildOpf(allocator: std.mem.Allocator, opts: Options) ![]u8 { + return std.fmt.allocPrint(allocator, + \\<?xml version="1.0" encoding="utf-8"?> + \\<package xmlns="http://www.idpf.org/2007/opf" version="3.0" unique-identifier="pub-id"> + \\ <metadata xmlns:dc="http://purl.org/dc/elements/1.1/"> + \\ <dc:identifier id="pub-id">urn:ziglang-docs:{s}</dc:identifier> + \\ <dc:title>{s} ({s})</dc:title> + \\ <dc:language>{s}</dc:language> + \\ <meta property="dcterms:modified">{s}</meta> + \\ </metadata> + \\ <manifest> + \\ <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav"/> + \\ <item id="index" href="index.xhtml" media-type="application/xhtml+xml"/> + \\ </manifest> + \\ <spine> + \\ <itemref idref="index"/> + \\ </spine> + \\</package> + \\ + , .{ opts.version, opts.title, opts.version, opts.lang, modified }); +} + +fn buildNav(allocator: std.mem.Allocator, opts: Options) ![]u8 { + return std.fmt.allocPrint(allocator, + \\<?xml version="1.0" encoding="utf-8"?> + \\<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops" lang="{s}"> + \\ <head><title>{s} ({s})</title></head> + \\ <body> + \\ <nav epub:type="toc" id="toc"> + \\ <h1>Contents</h1> + \\ <ol> + \\ <li><a href="index.xhtml">{s} ({s})</a></li> + \\ </ol> + \\ </nav> + \\ </body> + \\</html> + \\ + , .{ opts.lang, opts.title, opts.version, opts.title, opts.version }); +} + +test "epub starts with PK and mimetype, and round-trips" { + const gpa = std.testing.allocator; + const bytes = try build(gpa, .{ + .title = "Zig Language Reference", + .version = "0.15.2", + .xhtml = "<?xml version=\"1.0\"?>\n<html xmlns=\"http://www.w3.org/1999/xhtml\"><body>hi</body></html>", + }); + defer gpa.free(bytes); + + try std.testing.expect(std.mem.startsWith(u8, bytes, "PK")); + // mimetype must be the first entry, stored, with its content right after the + // local header + filename (offset 30 + len("mimetype") == 38). + try std.testing.expectEqualStrings("mimetype", bytes[30..38]); + try std.testing.expectEqualStrings("application/epub+zip", bytes[38..58]); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + try tmp.dir.writeFile(.{ .sub_path = "b.epub", .data = bytes }); + var file = try tmp.dir.openFile("b.epub", .{}); + defer file.close(); + var buf: [4096]u8 = undefined; + var fr = file.reader(&buf); + try std.zip.extract(tmp.dir, &fr, .{}); + + const opf = try tmp.dir.readFileAlloc(gpa, "OEBPS/content.opf", 1 << 16); + defer gpa.free(opf); + try std.testing.expect(std.mem.indexOf(u8, opf, "0.15.2") != null); +} diff --git a/src/fetch.zig b/src/fetch.zig new file mode 100644 index 0000000..706a435 --- /dev/null +++ b/src/fetch.zig @@ -0,0 +1,30 @@ +//! Thin HTTP GET helper over std.http.Client. + +const std = @import("std"); +const assert = std.debug.assert; + +/// Largest response body we accept (the docs page is ~1 MB; allow generous slack). +pub const max_body = 32 * 1024 * 1024; + +pub const Error = error{ HttpStatus, BodyTooLarge } || std.mem.Allocator.Error; + +/// GET `url` and return the response body. Caller owns the returned slice. +/// Returns an error on any non-200 status so callers can skip that version. +pub fn get(allocator: std.mem.Allocator, client: *std.http.Client, url: []const u8) ![]u8 { + assert(url.len > 0); + + var body: std.Io.Writer.Allocating = .init(allocator); + defer body.deinit(); + + const result = try client.fetch(.{ + .location = .{ .url = url }, + .response_writer = &body.writer, + }); + + if (result.status != .ok) return Error.HttpStatus; + if (body.writer.end > max_body) return Error.BodyTooLarge; + + const out = try body.toOwnedSlice(); + assert(out.len > 0); + return out; +} diff --git a/src/html.zig b/src/html.zig new file mode 100644 index 0000000..56c5d91 --- /dev/null +++ b/src/html.zig @@ -0,0 +1,309 @@ +//! Transform the Zig docs single-page HTML5 into well-formed XHTML5 for EPUB. +//! +//! The page is self-contained (embedded <style>, inline base64 icons, in-page TOC, +//! entity-escaped code). The only well-formedness work needed is: +//! 1. replace the leading `<!DOCTYPE html>` with an XML declaration, +//! 2. ensure the <html> element carries the XHTML namespace, +//! 3. strip <script>...</script> blocks (non-functional in EPUB, aids validation), +//! 4. normalize HTML named entities (e.g. `—`) to numeric references, +//! since XML predefines only `& < > " '`, +//! 5. self-close void elements (`<meta>` -> `<meta/>`, etc.). + +const std = @import("std"); +const assert = std.debug.assert; + +const xml_decl = "<?xml version=\"1.0\" encoding=\"utf-8\"?>\n"; +const xhtml_ns = "http://www.w3.org/1999/xhtml"; + +/// HTML void elements: never have children, must be self-closed in XHTML. +const void_elements = [_][]const u8{ + "meta", "link", "br", "hr", "img", "col", "input", "source", "area", "base", "wbr", "track", "param", +}; + +/// Transform `page` (HTML5) into well-formed XHTML5. Caller owns the result. +pub fn toXhtml(allocator: std.mem.Allocator, page: []const u8) ![]u8 { + assert(page.len > 0); + + const no_doctype = stripDoctype(page); + const no_scripts = try stripScripts(allocator, no_doctype); + defer allocator.free(no_scripts); + + const escaped = try escapeEntities(allocator, no_scripts); + defer allocator.free(escaped); + + var out: std.Io.Writer.Allocating = .init(allocator); + defer out.deinit(); + try out.writer.writeAll(xml_decl); + try writeWithFixups(&out.writer, escaped); + + const result = try out.toOwnedSlice(); + assert(std.mem.startsWith(u8, result, "<?xml")); + return result; +} + +/// Drop a leading `<!doctype html>` (case-insensitive), preserving the rest. +fn stripDoctype(page: []const u8) []const u8 { + const start = std.mem.indexOfNone(u8, page, " \t\r\n") orelse 0; + const rest = page[start..]; + if (rest.len >= 9 and std.ascii.eqlIgnoreCase(rest[0..9], "<!doctype")) { + const end = std.mem.indexOfScalar(u8, rest, '>') orelse return rest; + return rest[end + 1 ..]; + } + return page; +} + +/// Remove every `<script ...>...</script>` block. Caller owns the result. +fn stripScripts(allocator: std.mem.Allocator, html: []const u8) ![]u8 { + var out: std.Io.Writer.Allocating = .init(allocator); + defer out.deinit(); + + var i: usize = 0; + var guard: usize = 0; + while (i < html.len) { + guard += 1; + assert(guard <= html.len + 1); + + const open = findTagOpen(html, i, "script") orelse { + try out.writer.writeAll(html[i..]); + break; + }; + try out.writer.writeAll(html[i..open]); + + const close = findClose(html, open, "</script>") orelse { + // Unterminated script: drop the remainder rather than emit broken markup. + break; + }; + i = close; + } + return out.toOwnedSlice(); +} + +/// XML predefines only these five entities; everything else must be numeric. +const xml_entities = [_][]const u8{ "amp", "lt", "gt", "quot", "apos" }; + +/// Common HTML named entities → Unicode code point. Not exhaustive; any name not +/// listed (and not XML-predefined or numeric) has its `&` escaped to `&` so +/// output stays well-formed regardless of input. +const named_entities = std.StaticStringMap(u21).initComptime(.{ + .{ "nbsp", 0x00A0 }, .{ "copy", 0x00A9 }, .{ "reg", 0x00AE }, .{ "trade", 0x2122 }, + .{ "mdash", 0x2014 }, .{ "ndash", 0x2013 }, .{ "hellip", 0x2026 }, .{ "deg", 0x00B0 }, + .{ "plusmn", 0x00B1 }, .{ "times", 0x00D7 }, .{ "divide", 0x00F7 }, .{ "micro", 0x00B5 }, + .{ "middot", 0x00B7 }, .{ "bull", 0x2022 }, .{ "dagger", 0x2020 }, .{ "sect", 0x00A7 }, + .{ "para", 0x00B6 }, .{ "laquo", 0x00AB }, .{ "raquo", 0x00BB }, .{ "lsquo", 0x2018 }, + .{ "rsquo", 0x2019 }, .{ "ldquo", 0x201C }, .{ "rdquo", 0x201D }, .{ "larr", 0x2190 }, + .{ "rarr", 0x2192 }, .{ "uarr", 0x2191 }, .{ "darr", 0x2193 }, .{ "harr", 0x2194 }, + .{ "le", 0x2264 }, .{ "ge", 0x2265 }, .{ "ne", 0x2260 }, .{ "infin", 0x221E }, + .{ "sum", 0x2211 }, .{ "prod", 0x220F }, .{ "radic", 0x221A }, .{ "asymp", 0x2248 }, + .{ "equiv", 0x2261 }, .{ "frac12", 0x00BD }, .{ "frac14", 0x00BC }, .{ "frac34", 0x00BE }, + .{ "euro", 0x20AC }, .{ "pound", 0x00A3 }, .{ "yen", 0x00A5 }, .{ "cent", 0x00A2 }, +}); + +/// Longest entity name we will look ahead for (e.g. "frac34"). +const max_entity_name = 8; + +/// Rewrite `&...;` tokens so the output is XML well-formed. Caller owns the result. +fn escapeEntities(allocator: std.mem.Allocator, html: []const u8) ![]u8 { + var out: std.Io.Writer.Allocating = .init(allocator); + defer out.deinit(); + + var i: usize = 0; + while (i < html.len) { + const amp = std.mem.indexOfScalarPos(u8, html, i, '&') orelse { + try out.writer.writeAll(html[i..]); + break; + }; + try out.writer.writeAll(html[i..amp]); + + if (entityToken(html, amp)) |tok| { + if (isKeptEntity(tok)) { + try out.writer.writeAll(html[amp .. amp + tok.len + 2]); // include & and ; + } else if (named_entities.get(tok)) |cp| { + try out.writer.print("&#{d};", .{cp}); + } else { + try out.writer.writeAll("&"); + i = amp + 1; + continue; + } + i = amp + tok.len + 2; + } else { + // Bare '&' not forming an entity reference. + try out.writer.writeAll("&"); + i = amp + 1; + } + } + return out.toOwnedSlice(); +} + +/// If an `&...;` reference starts at `amp`, return the name between `&` and `;`. +fn entityToken(html: []const u8, amp: usize) ?[]const u8 { + assert(html[amp] == '&'); + const max = @min(html.len, amp + 2 + max_entity_name); + const semi = std.mem.indexOfScalarPos(u8, html[0..max], amp + 1, ';') orelse return null; + const tok = html[amp + 1 .. semi]; + if (tok.len == 0) return null; + return tok; +} + +/// True for entities XML keeps verbatim: the predefined five and numeric refs. +fn isKeptEntity(tok: []const u8) bool { + if (tok[0] == '#') { + const digits = tok[1..]; + if (digits.len == 0) return false; + const hex = digits[0] == 'x' or digits[0] == 'X'; + const rest = if (hex) digits[1..] else digits; + if (rest.len == 0) return false; + for (rest) |c| { + const ok = if (hex) std.ascii.isHex(c) else std.ascii.isDigit(c); + if (!ok) return false; + } + return true; + } + for (xml_entities) |e| { + if (std.mem.eql(u8, tok, e)) return true; + } + return false; +} + +/// Copy `html` to `w`, adding the XHTML namespace to <html> and self-closing +/// void elements as they are encountered. +fn writeWithFixups(w: *std.Io.Writer, html: []const u8) !void { + var i: usize = 0; + while (i < html.len) { + const lt = std.mem.indexOfScalarPos(u8, html, i, '<') orelse { + try w.writeAll(html[i..]); + return; + }; + try w.writeAll(html[i..lt]); + + const gt = std.mem.indexOfScalarPos(u8, html, lt, '>') orelse { + try w.writeAll(html[lt..]); + return; + }; + const tag = html[lt .. gt + 1]; // includes < and > + try writeTag(w, tag); + i = gt + 1; + } +} + +/// Emit a single tag (`<...>`), applying namespace/void-element fixups. +fn writeTag(w: *std.Io.Writer, tag: []const u8) !void { + assert(tag.len >= 2); + assert(tag[0] == '<' and tag[tag.len - 1] == '>'); + + const name = tagName(tag); + const is_closing = tag.len >= 2 and tag[1] == '/'; + + if (!is_closing and std.ascii.eqlIgnoreCase(name, "html") and std.mem.indexOf(u8, tag, "xmlns") == null) { + try w.print("<html xmlns=\"{s}\"", .{xhtml_ns}); + try w.writeAll(tag[1 + name.len ..]); + return; + } + + if (isVoid(name) and !std.mem.endsWith(u8, tag, "/>")) { + try w.writeAll(tag[0 .. tag.len - 1]); + try w.writeAll("/>"); + return; + } + + try w.writeAll(tag); +} + +/// Extract the element name from a tag slice (without `<`, `>`, `/`, attrs). +fn tagName(tag: []const u8) []const u8 { + var s: usize = 1; // skip '<' + if (s < tag.len and tag[s] == '/') s += 1; + var e = s; + while (e < tag.len and isNameChar(tag[e])) e += 1; + return tag[s..e]; +} + +fn isNameChar(c: u8) bool { + return std.ascii.isAlphanumeric(c) or c == '-' or c == ':'; +} + +fn isVoid(name: []const u8) bool { + for (void_elements) |v| { + if (std.ascii.eqlIgnoreCase(name, v)) return true; + } + return false; +} + +/// Find the next `<name` opening tag at or after `from`, returning the `<` index. +fn findTagOpen(html: []const u8, from: usize, name: []const u8) ?usize { + var i = from; + while (std.mem.indexOfScalarPos(u8, html, i, '<')) |lt| { + const after = lt + 1; + if (after + name.len <= html.len and + std.ascii.eqlIgnoreCase(html[after .. after + name.len], name)) + { + const next = html[after + name.len]; + if (next == '>' or next == ' ' or next == '\t' or next == '\n' or next == '/') return lt; + } + i = lt + 1; + } + return null; +} + +/// Find the index just past a closing tag (e.g. `</script>`) at or after `from`. +fn findClose(html: []const u8, from: usize, close: []const u8) ?usize { + const idx = std.ascii.indexOfIgnoreCasePos(html, from, close) orelse return null; + return idx + close.len; +} + +test "strips doctype and adds xml declaration" { + const gpa = std.testing.allocator; + const out = try toXhtml(gpa, "<!DOCTYPE html>\n<html><body>x</body></html>"); + defer gpa.free(out); + try std.testing.expect(std.mem.startsWith(u8, out, "<?xml")); + try std.testing.expect(std.mem.indexOf(u8, out, "<!DOCTYPE") == null); +} + +test "adds xhtml namespace to html element" { + const gpa = std.testing.allocator; + const out = try toXhtml(gpa, "<html lang=\"en\"><head></head></html>"); + defer gpa.free(out); + try std.testing.expect(std.mem.indexOf(u8, out, "xmlns=\"http://www.w3.org/1999/xhtml\"") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "lang=\"en\"") != null); + // The closing tag must be left intact (not rewritten with a namespace). + try std.testing.expect(std.mem.endsWith(u8, out, "</html>")); + try std.testing.expect(std.mem.indexOf(u8, out, "xhtml\"l>") == null); +} + +test "self-closes void elements" { + const gpa = std.testing.allocator; + const out = try toXhtml(gpa, "<html><head><meta charset=\"utf-8\"><br></head></html>"); + defer gpa.free(out); + try std.testing.expect(std.mem.indexOf(u8, out, "<meta charset=\"utf-8\"/>") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "<br/>") != null); +} + +test "strips script blocks" { + const gpa = std.testing.allocator; + const out = try toXhtml(gpa, "<html><body>a<script>var x = 1 < 2;</script>b</body></html>"); + defer gpa.free(out); + try std.testing.expect(std.mem.indexOf(u8, out, "<script") == null); + try std.testing.expect(std.mem.indexOf(u8, out, "var x") == null); + try std.testing.expect(std.mem.indexOf(u8, out, ">a") != null); + try std.testing.expect(std.mem.indexOf(u8, out, "b<") != null); +} + +test "normalizes named entities and escapes stray ampersands" { + const gpa = std.testing.allocator; + const out = try toXhtml(gpa, "<html><body>a—b & c d R&D <x> — &unknownent;</body></html>"); + defer gpa.free(out); + try std.testing.expect(std.mem.indexOf(u8, out, "—") == null); + try std.testing.expect(std.mem.indexOf(u8, out, "—") != null); // mdash -> numeric + try std.testing.expect(std.mem.indexOf(u8, out, " ") != null); // nbsp -> numeric + try std.testing.expect(std.mem.indexOf(u8, out, "R&D") != null); // stray & escaped + try std.testing.expect(std.mem.indexOf(u8, out, "&unknownent;") != null); // unknown name escaped + try std.testing.expect(std.mem.indexOf(u8, out, "& c") != null); // existing & kept (not double-escaped) + try std.testing.expect(std.mem.indexOf(u8, out, "<x>") != null); +} + +test "leaves already self-closed void elements unchanged" { + const gpa = std.testing.allocator; + const out = try toXhtml(gpa, "<html><head><meta charset=\"utf-8\"/></head></html>"); + defer gpa.free(out); + try std.testing.expect(std.mem.indexOf(u8, out, "<meta charset=\"utf-8\"//>") == null); + try std.testing.expect(std.mem.indexOf(u8, out, "<meta charset=\"utf-8\"/>") != null); +} diff --git a/src/index.zig b/src/index.zig new file mode 100644 index 0000000..7eb3e4b --- /dev/null +++ b/src/index.zig @@ -0,0 +1,103 @@ +//! Discover Zig versions and their docs URLs from ziglang.org/download/index.json. + +const std = @import("std"); +const assert = std.debug.assert; +const fetch = @import("fetch.zig"); + +pub const index_url = "https://ziglang.org/download/index.json"; + +/// Upper bound on versions we will process (NASA Power-of-10: bounded loops). +pub const max_versions = 64; + +pub const Version = struct { + /// Release name as it appears in index.json (e.g. "master", "0.15.2"). + name: []const u8, + /// Absolute URL of the single-page language reference. + docs_url: []const u8, +}; + +/// Owns the slice of versions and all their backing strings. +pub const Versions = struct { + allocator: std.mem.Allocator, + items: []Version, + + pub fn deinit(self: *Versions) void { + for (self.items) |v| { + self.allocator.free(v.name); + self.allocator.free(v.docs_url); + } + self.allocator.free(self.items); + } +}; + +/// Fetch and parse index.json, collecting every entry that has a `docs` URL. +pub fn fetchAll(allocator: std.mem.Allocator, client: *std.http.Client) !Versions { + const json = try fetch.get(allocator, client, index_url); + defer allocator.free(json); + return parse(allocator, json); +} + +/// Parse index.json bytes into owned versions. Split out for testability. +pub fn parse(allocator: std.mem.Allocator, json: []const u8) !Versions { + assert(json.len > 0); + + var parsed = try std.json.parseFromSlice(std.json.Value, allocator, json, .{}); + defer parsed.deinit(); + + const root = switch (parsed.value) { + .object => |o| o, + else => return error.MalformedIndex, + }; + + var list: std.ArrayList(Version) = .empty; + errdefer { + for (list.items) |v| { + allocator.free(v.name); + allocator.free(v.docs_url); + } + list.deinit(allocator); + } + + var it = root.iterator(); + while (it.next()) |entry| { + if (list.items.len >= max_versions) break; + + const obj = switch (entry.value_ptr.*) { + .object => |o| o, + else => continue, + }; + const docs = switch (obj.get("docs") orelse continue) { + .string => |s| s, + else => continue, + }; + if (docs.len == 0) continue; + + const name = try allocator.dupe(u8, entry.key_ptr.*); + errdefer allocator.free(name); + const url = try allocator.dupe(u8, docs); + try list.append(allocator, .{ .name = name, .docs_url = url }); + } + + return .{ .allocator = allocator, .items = try list.toOwnedSlice(allocator) }; +} + +test "parses versions with docs urls and skips entries without" { + const gpa = std.testing.allocator; + const json = + \\{ + \\ "master": { "version": "0.16.0-dev", "docs": "https://ziglang.org/documentation/master/" }, + \\ "0.15.2": { "docs": "https://ziglang.org/documentation/0.15.2/" }, + \\ "0.1.0": { "notes": "no docs here" } + \\} + ; + var versions = try parse(gpa, json); + defer versions.deinit(); + + try std.testing.expectEqual(@as(usize, 2), versions.items.len); + var saw_master = false; + for (versions.items) |v| { + try std.testing.expect(v.docs_url.len > 0); + if (std.mem.eql(u8, v.name, "master")) saw_master = true; + } + try std.testing.expect(saw_master); +} diff --git a/src/main.zig b/src/main.zig new file mode 100644 index 0000000..a6bbe31 --- /dev/null +++ b/src/main.zig @@ -0,0 +1,106 @@ +//! ziglang-docs-epub: turn the Zig language reference into one EPUB per version. +//! +//! Usage: +//! ziglang-docs-epub [version...] +//! no args build every version in index.json that has a docs URL +//! version build only the named version(s), e.g. `0.15.2 master` +//! +//! Output is written to the `epubs/` directory as `zig-<version>.epub`. + +const std = @import("std"); +const assert = std.debug.assert; + +const fetch = @import("fetch.zig"); +const index = @import("index.zig"); +const html = @import("html.zig"); +const epub = @import("epub.zig"); + +const out_dir = "epubs"; +const title = "Zig Language Reference"; +/// A valid EPUB is always larger than this; used to sanity-check output. +const min_epub_size = 256; + +pub fn main() !void { + var gpa_state: std.heap.GeneralPurposeAllocator(.{}) = .init; + defer _ = gpa_state.deinit(); + const gpa = gpa_state.allocator(); + + const args = try std.process.argsAlloc(gpa); + defer std.process.argsFree(gpa, args); + const filters = args[1..]; // empty == build all + + var client: std.http.Client = .{ .allocator = gpa }; + defer client.deinit(); + + var versions = index.fetchAll(gpa, &client) catch |err| { + std.log.err("failed to fetch version index: {s}", .{@errorName(err)}); + return err; + }; + defer versions.deinit(); + + var dir = try std.fs.cwd().makeOpenPath(out_dir, .{}); + defer dir.close(); + + var built: usize = 0; + var skipped: usize = 0; + + assert(versions.items.len <= index.max_versions); + for (versions.items) |v| { + if (!wanted(filters, v.name)) continue; + + buildVersion(gpa, &client, dir, v) catch |err| { + std.log.warn("skip {s}: {s}", .{ v.name, @errorName(err) }); + skipped += 1; + continue; + }; + built += 1; + } + + std.log.info("done: {d} built, {d} skipped", .{ built, skipped }); + if (built == 0) std.log.warn("no epubs were produced", .{}); +} + +/// Fetch, transform, package and write the EPUB for a single version. +fn buildVersion( + gpa: std.mem.Allocator, + client: *std.http.Client, + dir: std.fs.Dir, + v: index.Version, +) !void { + const page = try fetch.get(gpa, client, v.docs_url); + defer gpa.free(page); + + const xhtml = try html.toXhtml(gpa, page); + defer gpa.free(xhtml); + + const bytes = try epub.build(gpa, .{ .title = title, .version = v.name, .xhtml = xhtml }); + defer gpa.free(bytes); + + assert(std.mem.startsWith(u8, bytes, "PK")); + assert(bytes.len > min_epub_size); + + const name = try std.fmt.allocPrint(gpa, "zig-{s}.epub", .{v.name}); + defer gpa.free(name); + try dir.writeFile(.{ .sub_path = name, .data = bytes }); + + std.log.info("built {s} ({d} bytes)", .{ name, bytes.len }); +} + +/// True if `name` should be built given the CLI `filters` (empty == all). +fn wanted(filters: []const []const u8, name: []const u8) bool { + if (filters.len == 0) return true; + for (filters) |f| { + if (std.mem.eql(u8, f, name)) return true; + } + return false; +} + +test "wanted matches filters or accepts all when empty" { + try std.testing.expect(wanted(&.{}, "0.15.2")); + try std.testing.expect(wanted(&.{"0.15.2"}, "0.15.2")); + try std.testing.expect(!wanted(&.{"master"}, "0.15.2")); +} + +test { + std.testing.refAllDecls(@This()); +} diff --git a/src/zip.zig b/src/zip.zig new file mode 100644 index 0000000..9268106 --- /dev/null +++ b/src/zip.zig @@ -0,0 +1,192 @@ +//! Minimal ZIP writer: stored (uncompressed) entries, central directory, EOCD. +//! +//! Entries are stored uncompressed (method 0). Zig 0.15.2's std deflate +//! *compressor* is unfinished (`std.compress.flate.Compress` panics / loops), +//! so we avoid it entirely; stored entries are fully ZIP/EPUB compliant. The +//! cost is larger files, acceptable for docs committed to the repo. +//! +//! Output is byte-for-byte reproducible: a fixed DOS timestamp is used for every +//! entry (no wall clock), so identical inputs produce identical archives and git +//! only sees a diff when the underlying docs actually change. + +const std = @import("std"); +const assert = std.debug.assert; + +const Crc32 = std.hash.crc.Crc32; + +/// Upper bound on entries in a single archive (NASA Power-of-10: bounded loops). +pub const max_entries = 64; + +/// Fixed DOS date/time for reproducibility (2021-01-01 00:00:00). +const dos_time: u16 = 0; +const dos_date: u16 = (41 << 9) | (1 << 5) | 1; // (year-1980)<<9 | month<<5 | day + +const sig_local = std.zip.local_file_header_sig; +const sig_central = std.zip.central_file_header_sig; +const sig_end = std.zip.end_record_sig; + +const method_store: u16 = @intFromEnum(std.zip.CompressionMethod.store); + +const Entry = struct { + name: []const u8, + crc: u32, + comp_size: u32, + uncomp_size: u32, + method: u16, + offset: u32, +}; + +pub const Writer = struct { + allocator: std.mem.Allocator, + bytes: std.ArrayList(u8), + entries: std.ArrayList(Entry), + + pub fn init(allocator: std.mem.Allocator) Writer { + return .{ + .allocator = allocator, + .bytes = .empty, + .entries = .empty, + }; + } + + pub fn deinit(w: *Writer) void { + w.bytes.deinit(w.allocator); + w.entries.deinit(w.allocator); + } + + /// Add an entry stored uncompressed (method 0). + pub fn addStored(w: *Writer, name: []const u8, data: []const u8) !void { + assert(name.len > 0); + assert(w.entries.items.len < max_entries); + const crc = Crc32.hash(data); + try w.writeLocal(name, crc, @intCast(data.len), @intCast(data.len), method_store); + try w.bytes.appendSlice(w.allocator, data); + } + + /// Append central directory + end-of-central-directory; return archive bytes. + /// Caller owns the returned slice. + pub fn finish(w: *Writer) ![]u8 { + assert(w.entries.items.len > 0); + const cd_offset: u32 = @intCast(w.bytes.items.len); + + for (w.entries.items) |e| { + try w.appendBytes(&sig_central); + try w.appendInt(u16, 20); // version made by + try w.appendInt(u16, 20); // version needed + try w.appendInt(u16, 0); // flags + try w.appendInt(u16, e.method); + try w.appendInt(u16, dos_time); + try w.appendInt(u16, dos_date); + try w.appendInt(u32, e.crc); + try w.appendInt(u32, e.comp_size); + try w.appendInt(u32, e.uncomp_size); + try w.appendInt(u16, @intCast(e.name.len)); + try w.appendInt(u16, 0); // extra len + try w.appendInt(u16, 0); // comment len + try w.appendInt(u16, 0); // disk number start + try w.appendInt(u16, 0); // internal attrs + try w.appendInt(u32, 0); // external attrs + try w.appendInt(u32, e.offset); + try w.appendBytes(e.name); + } + + const cd_size: u32 = @as(u32, @intCast(w.bytes.items.len)) - cd_offset; + const count: u16 = @intCast(w.entries.items.len); + + try w.appendBytes(&sig_end); + try w.appendInt(u16, 0); // disk number + try w.appendInt(u16, 0); // cd start disk + try w.appendInt(u16, count); // records this disk + try w.appendInt(u16, count); // total records + try w.appendInt(u32, cd_size); + try w.appendInt(u32, cd_offset); + try w.appendInt(u16, 0); // comment len + + return w.bytes.toOwnedSlice(w.allocator); + } + + fn writeLocal(w: *Writer, name: []const u8, crc: u32, comp_size: u32, uncomp_size: u32, method: u16) !void { + const offset: u32 = @intCast(w.bytes.items.len); + assert(w.entries.items.len == 0 or offset > w.entries.items[w.entries.items.len - 1].offset); + + try w.appendBytes(&sig_local); + try w.appendInt(u16, 20); // version needed + try w.appendInt(u16, 0); // flags + try w.appendInt(u16, method); + try w.appendInt(u16, dos_time); + try w.appendInt(u16, dos_date); + try w.appendInt(u32, crc); + try w.appendInt(u32, comp_size); + try w.appendInt(u32, uncomp_size); + try w.appendInt(u16, @intCast(name.len)); + try w.appendInt(u16, 0); // extra len + try w.appendBytes(name); + + try w.entries.append(w.allocator, .{ + .name = name, + .crc = crc, + .comp_size = comp_size, + .uncomp_size = uncomp_size, + .method = method, + .offset = offset, + }); + } + + fn appendBytes(w: *Writer, data: []const u8) !void { + try w.bytes.appendSlice(w.allocator, data); + } + + fn appendInt(w: *Writer, comptime T: type, value: T) !void { + var buf: [@sizeOf(T)]u8 = undefined; + std.mem.writeInt(T, &buf, value, .little); + try w.bytes.appendSlice(w.allocator, &buf); + } +}; + +/// Write `archive` to a temp dir, extract it with the std.zip reader, and return +/// the extracted contents of `name`. Exercises a full ZIP round-trip. +fn extractOne(gpa: std.mem.Allocator, dir: std.fs.Dir, archive: []const u8, name: []const u8) ![]u8 { + try dir.writeFile(.{ .sub_path = "out.zip", .data = archive }); + var file = try dir.openFile("out.zip", .{}); + defer file.close(); + var buf: [4096]u8 = undefined; + var fr = file.reader(&buf); + try std.zip.extract(dir, &fr, .{}); + return dir.readFileAlloc(gpa, name, 1 << 20); +} + +test "stored entry round-trips via std.zip reader" { + const gpa = std.testing.allocator; + var w = Writer.init(gpa); + defer w.deinit(); + + const payload = "hello, epub"; + try w.addStored("a.txt", payload); + const archive = try w.finish(); + defer gpa.free(archive); + + try std.testing.expect(std.mem.startsWith(u8, archive, "PK")); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const got = try extractOne(gpa, tmp.dir, archive, "a.txt"); + defer gpa.free(got); + try std.testing.expectEqualStrings(payload, got); +} + +test "multiple entries round-trip in nested paths" { + const gpa = std.testing.allocator; + var w = Writer.init(gpa); + defer w.deinit(); + + try w.addStored("first.txt", "one"); + try w.addStored("dir/second.txt", "A" ** 1000); + const archive = try w.finish(); + defer gpa.free(archive); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const got = try extractOne(gpa, tmp.dir, archive, "dir/second.txt"); + defer gpa.free(got); + try std.testing.expectEqualStrings("A" ** 1000, got); +} |