From 870c8cb01258ca0b0d28cf00a80bba9f6e69d4b8 Mon Sep 17 00:00:00 2001 From: mr-boneman Date: Tue, 20 Aug 2024 14:29:55 +0200 Subject: [PATCH] http server --- .gitignore | 1 + Containerfile | 46 +++++++++++- config.nims | 1 - configs/fedora.toml | 9 +++ configs/releases.toml | 9 +++ docker-compose.yml | 7 ++ justfile | 17 +++++ src/rss.nim | 66 ++++++++++++++++ src/rssmix.nim | 171 +++++++++++++++++++++++------------------- 9 files changed, 246 insertions(+), 81 deletions(-) delete mode 100644 config.nims create mode 100644 configs/fedora.toml create mode 100644 configs/releases.toml create mode 100644 docker-compose.yml create mode 100644 justfile create mode 100644 src/rss.nim diff --git a/.gitignore b/.gitignore index d955720..8d713a7 100644 --- a/.gitignore +++ b/.gitignore @@ -17,3 +17,4 @@ htmldocs/ # Built Visual Studio Code Extensions *.vsix +rssmix \ No newline at end of file diff --git a/Containerfile b/Containerfile index 0dfefc1..011f380 100644 --- a/Containerfile +++ b/Containerfile @@ -1,2 +1,44 @@ -ARG fedoraVersion=40 -FROM quay.io/fedora/fedora-minimal:${fedoraVersion} \ No newline at end of file +FROM quay.io/fedora/fedora-minimal:40 AS builder + +RUN dnf5 install -y wget \ + gcc glibc-devel just \ + git tar xz libcurl-devel + +RUN printf ". /etc/bashrc\n" > $HOME/.bashrc + +ARG nimVersion=2.0.8 +RUN set -euxo pipefail && \ + tempdir=$(mktemp -d) && \ + pushd $tempdir && \ + curl -o nim.tar.xz https://nim-lang.org/download/nim-${nimVersion}-linux_x64.tar.xz && \ + tar -xf nim.tar.xz && \ + pushd nim-${nimVersion} && \ + ./install.sh /usr/bin && \ + popd && \ + cp nim-${nimVersion}/bin/* /usr/bin && \ + mkdir ./lib && mv /usr/lib/nim/* ./lib && mv ./lib /usr/lib/nim/lib # fixes nim not finding the system lib && \ + printf "export PATH=$HOME/.nimble/bin:'$PATH'" >> $HOME/.bashrc && \ + popd && \ + rm -fr $tempdir + +WORKDIR /workdir +COPY . /workdir + +ARG NIMBLESETTINGS="" +RUN just build-binary '${NIMBLESETTINGS}' + + +FROM quay.io/fedora/fedora-minimal:40 + +# Add Tini +ENV TINI_VERSION v0.19.0 +ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] + +RUN dnf5 -y install libcurl && dnf5 clean all + +COPY --from=builder /workdir/rssmix /bin/rssmix + +STOPSIGNAL SIGINT +CMD [ "/bin/rssmix" ] \ No newline at end of file diff --git a/config.nims b/config.nims deleted file mode 100644 index ca97df1..0000000 --- a/config.nims +++ /dev/null @@ -1 +0,0 @@ -switch("d", "ssl") \ No newline at end of file diff --git a/configs/fedora.toml b/configs/fedora.toml new file mode 100644 index 0000000..047330d --- /dev/null +++ b/configs/fedora.toml @@ -0,0 +1,9 @@ +name = "fedora" + +[[feeds]] +url = "https://fedoramagazine.org/rss" +kind = "RSSv2" + +[[feeds]] +url = "http://communityblog.fedoraproject.org/?feed=rss2" +kind = "RSSv2" \ No newline at end of file diff --git a/configs/releases.toml b/configs/releases.toml new file mode 100644 index 0000000..d2673d7 --- /dev/null +++ b/configs/releases.toml @@ -0,0 +1,9 @@ +name = "releases" + +[[feeds]] +url = "https://github.com/nim-lang/Nim/releases.atom" +kind = "Atom" + +[[feeds]] +url = "https://github.com/fatedier/frp/releases.atom" +kind = "Atom" \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..954c606 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,7 @@ +services: + bimage: + image: rssmix:latest + ports: + - 8085:8085 + volumes: + - ./configs:/configs \ No newline at end of file diff --git a/justfile b/justfile new file mode 100644 index 0000000..6659ff9 --- /dev/null +++ b/justfile @@ -0,0 +1,17 @@ +all: dev + +dev nimblesettings="": + nimble run {{nimblesettings}} rssmix configs + +build-dev nimblesettings="": + nimble build {{nimblesettings}} rssmix + +build-binary nimblesettings="": + nimble -y {{nimblesettings}} build rssmix + strip rssmix + +build buildahsettings="" buildahbuildsettings="" nimblesettings="-d:defaultConfigPath:/configs -d:release --opt:speed -d:strip": + buildah {{buildahsettings}} build {{buildahbuildsettings}} -t rssmix:latest -f Containerfile --build-arg=NIMBLESETTINGS='{{nimblesettings}}' . + +copy-to-docker: + skopeo copy containers-storage:localhost/rssmix:latest docker-daemon:rssmix:latest diff --git a/src/rss.nim b/src/rss.nim new file mode 100644 index 0000000..8062491 --- /dev/null +++ b/src/rss.nim @@ -0,0 +1,66 @@ +import std/[algorithm, json, xmltree, times, options, sequtils] +import pkg/[mummy, rssatom, puppy, pretty] + +const + atomTimeFormat* {.strdefine.} = "yyyy-MM-dd'T'HH:mm:ss'Z'" + rssv2TimeFormat* {.strdefine.} = "ddd',' dd MMM yyyy HH:mm:ss ZZZ" + +type RssFeed* = object # name:string + url*: string + kind*: FeedType + +proc parseAtomTime(s: string): DateTime = + parse(s, atomTimeFormat) + +proc parseRSSv2Time(s: string): DateTime = + parse(s, rssv2TimeFormat) + +proc getrssv2Time(r: RssItem): DateTime = + result = dateTime(0, Month.low, MonthdayRange.low) + if r.pubDate.isSome(): + result = parseRssv2Time(r.pubDate.get()) + +proc getTime(r: RssItem): DateTime = + result = dateTime(0, Month.low, MonthdayRange.low) + if r.updated.isSome(): + result = parseAtomTime(r.updated.get()) + elif r.pubDate.isSome(): + result = parseAtomTime(r.pubDate.get()) + +proc cmpRssItem(x, y: RssItem): int = + cmp(x.getTime(), y.getTime()) + +proc mixRssFeeds*(feeds: seq[RssFeed], name, id, link, authorName, description:string): RSS = + result = RSS() + result.id = name.some() + result.title = id.some() + result.link = link.some() + result.author.name = authorName.some() + result.description = description.some() + var entries: seq[RSS] + for feed in feeds: + case feed.kind + of Atom: + entries.add parseAtom(fetch(feed.url)) + of RSSv2: + entries.add parseRss(fetch(feed.url)) + + entries[^1].items = entries[^1].items.mapIt( + block: + var item = it + item.pubdate = some(getRssv2Time(item).format(atomTimeFormat)) + item + ) + + for feed in entries: + result.items.add feed.items + + result.items.sort(cmp = cmpRssItem, order = Descending) + +proc assembleAtom*(r: RSS): XmlNode = + result = buildAtom(r) + result.attrs = { + "xmlns": "http://www.w3.org/2005/Atom", + "xmlns:thr": "http://purl.org/syndication/thread/1.0", + "xml:lang": "en-EN", + }.toXmlAttributes() diff --git a/src/rssmix.nim b/src/rssmix.nim index 3f60da9..70060fa 100644 --- a/src/rssmix.nim +++ b/src/rssmix.nim @@ -1,80 +1,95 @@ -import std/[algorithm, json, xmltree, times, options, sequtils] -import pkg/[mummy, rssatom, puppy, pretty] - -const - atomTimeFormat {.strdefine.} = "yyyy-MM-dd'T'HH:mm:ss'Z'" - rssv2TimeFormat {.strdefine.} = "ddd',' dd MMM yyyy HH:mm:ss ZZZ" - -type RssFeed = object # name:string - url: string - kind: FeedType - -proc parseAtomTime(s: string): DateTime = - parse(s, atomTimeFormat) - -proc parseRSSv2Time(s: string): DateTime = - parse(s, rssv2TimeFormat) - -proc getrssv2Time(r:RssItem):DateTime= - result = dateTime(0, Month.low, MonthdayRange.low) - if r.pubDate.isSome(): - result = parseRssv2Time(r.pubDate.get()) - - -proc getTime(r: RssItem): DateTime = - result = dateTime(0, Month.low, MonthdayRange.low) - if r.updated.isSome(): - result = parseAtomTime(r.updated.get()) - elif r.pubDate.isSome(): - result = parseAtomTime(r.pubDate.get()) - -proc cmpRssItem(x, y: RssItem): int = - cmp(x.getTime(), y.getTime()) - -proc mixRssFeeds(feeds: seq[RssFeed]): RSS = - result = RSS() - result.id = "https://example.com/".some() - result.title = "This is a test".some() - result.link = "link.com".some() - result.author.name = "Samuel R.".some() - result.description = "this is a test for this and that".some() - var entries: seq[RSS] - for feed in feeds: - case feed.kind - of Atom: - entries.add parseAtom(fetch(feed.url)) - of RSSv2: - entries.add parseRss(fetch(feed.url)) - - entries[^1].items = entries[^1].items.mapIt( - block: - var item = it - item.pubdate = some(getRssv2Time(item).format(atomTimeFormat)) - item - ) - - for feed in entries: - result.items.add feed.items - - result.items.sort(cmp = cmpRssItem, order = Descending) - -proc assembleAtom(r: RSS): XmlNode = - result = buildAtom(r) - result.attrs = { - "xmlns": "http://www.w3.org/2005/Atom", - "xmlns:thr": "http://purl.org/syndication/thread/1.0", - "xml:lang": "en-EN" - }.toXmlAttributes() - - discard - -echo assembleAtom( - mixRssFeeds( - @[ - RssFeed(kind: Atom, url: "https://github.com/nim-lang/Nim/releases.atom"), - RssFeed(kind: Atom, url: "https://github.com/fatedier/frp/releases.atom"), - RssFeed(kind: RSSv2, url: "https://fedoramagazine.org/rss"), - RssFeed(kind: RSSv2, url: "http://communityblog.fedoraproject.org/?feed=rss2"), +import + std/ + [ + algorithm, json, xmltree, times, options, sequtils, tables, os, exitprocs, + strutils, ] +import pkg/[rssatom, pretty, parsetoml, jsony] +import pkg/[mummy, mummy/routers] +import ./[tomlToJson, rss] + +const defaultConfigPath {.strdefine.} = "" + +type Config = object + name: string + feeds: seq[RssFeed] + +proc parseConfig(content: string): Config = + parseString(content).customtoJson().toJson().fromJson(Config) + +var + configs: Table[string, seq[RssFeed]] + server: Server + stopping: bool + +proc atomHandler(r: Request) {.gcsafe, thread.} = + let name = r.pathParams["name"] + {.gcsafe.}: + # safe as configs won't be modified after the server started + if not configs.contains(name): + r.respond(404, body = "no such mix") + return + var headers: HttpHeaders + headers["Content-Type"] = "application/atom+xml" + + var cfg = configs[name] + let mixed = mixRssFeeds(cfg, "rssmix: "&name, name, "/atom/"&name, "rssmix", "mix of various rss feeds") + + r.respond(200, headers, body = $assembleAtom(mixed)) + +proc stop() = + if stopping: + return + stopping = true + echo "closing server" + server.close() + quit() + +proc addfeeds(name:string, feeds:seq[RssFeed])= + if configs.contains(name): raise newException(KeyError, "duplicate config: "&name) + configs[name] = feeds + + +proc main() = + block: + var cfgs = commandLineParams() + if cfgs.len == 0: + when defaultConfigPath.len == 0: + quit("rssmix configs", QuitFailure) + else: + cfgs = @[defaultConfigPath] + for cfg in cfgs: + if fileExists(cfg): + let cfg = readFile(cfg).parseConfig() + addfeeds(cfg.name, cfg.feeds) + elif dirExists(cfg): + for f in walkFiles(cfg/"*"): + let cfg = readFile(f).parseConfig() + addfeeds(cfg.name, cfg.feeds) + if configs.len == 0: quit("no configs found.", QuitFailure) + echo "registered feeds:" + for key, value in configs.pairs: + echo " - ", key, " (", value.len, " sub-feeds)" + + + + var router: Router + router.get("/atom/@name", atomHandler) + + server = newServer(router) + + addExitProc(stop) + setControlCHook( + proc() {.noconv.} = + stop() ) -) + + let + bindAddr = getEnv("RSSMIX_BINDADDR", "0.0.0.0") + port = getEnv("RSSMIX_PORT", "8085") + + echo "running on http://" & bindAddr & ":" & port + server.serve(Port(parseInt(port)), bindAddr) + +when isMainModule: + main()