add rss logs, reorganize index

poetry.lock

@@ -64,6 +64,19 @@ soupsieve = [">1.2", "<2.0"]
 html5lib = ["html5lib"]
 lxml = ["lxml"]
 
+[[package]]
+category = "main"
+description = "An easy safelist-based HTML-sanitizing tool."
+name = "bleach"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*"
+version = "3.1.5"
+
+[package.dependencies]
+packaging = "*"
+six = ">=1.9.0"
+webencodings = "*"
+
 [[package]]
 category = "main"
 description = "Universal encoding detector for Python 2 and 3"
@@ -89,6 +102,18 @@ sqlparse = ">=0.2.2"
 argon2 = ["argon2-cffi (>=16.1.0)"]
 bcrypt = ["bcrypt"]
 
+[[package]]
+category = "main"
+description = "Easily use bleach with Django models and templates"
+name = "django-bleach"
+optional = false
+python-versions = "*"
+version = "0.6.1"
+
+[package.dependencies]
+Django = ">=1.11"
+bleach = ">=1.5.0"
+
 [[package]]
 category = "main"
 description = "Allows Django models to be ordered and provides a simple admin interface for reordering them."
@@ -97,6 +122,14 @@ optional = false
 python-versions = "*"
 version = "3.4.1"
 
+[[package]]
+category = "main"
+description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds"
+name = "feedparser"
+optional = false
+python-versions = "*"
+version = "5.2.1"
+
 [[package]]
 category = "main"
 description = "Python humanize utilities"
@@ -124,6 +157,18 @@ optional = false
 python-versions = ">=3.5"
 version = "4.7.6"
 
+[[package]]
+category = "main"
+description = "Core utilities for Python packages"
+name = "packaging"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
+version = "20.4"
+
+[package.dependencies]
+pyparsing = ">=2.0.2"
+six = "*"
+
 [[package]]
 category = "main"
 description = "Python Imaging Library (Fork)"
@@ -143,6 +188,14 @@ version = "5.7.0"
 [package.extras]
 enum = ["enum34"]
 
+[[package]]
+category = "main"
+description = "Python parsing module"
+name = "pyparsing"
+optional = false
+python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*"
+version = "2.4.7"
+
 [[package]]
 category = "main"
 description = "World timezone definitions, modern and historical"
@@ -151,6 +204,14 @@ optional = false
 python-versions = "*"
 version = "2020.1"
 
+[[package]]
+category = "main"
+description = "Python 2 and 3 compatibility utilities"
+name = "six"
+optional = false
+python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*"
+version = "1.15.0"
+
 [[package]]
 category = "main"
 description = "A modern CSS selector implementation for Beautiful Soup."
@@ -167,6 +228,14 @@ optional = false
 python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
 version = "0.3.1"
 
+[[package]]
+category = "main"
+description = "Character encoding aliases for legacy web content"
+name = "webencodings"
+optional = false
+python-versions = "*"
+version = "0.5.1"
+
 [[package]]
 category = "main"
 description = "Yet another URL library"
@@ -180,7 +249,7 @@ idna = ">=2.0"
 multidict = ">=4.0"
 
 [metadata]
-content-hash = "c17c9f2b361876b298b7a7a2917926979b5681ee7d037fbb2c19bfd8f5f3f352"
+content-hash = "5fcbd04cee09c4e0ec32a23ab7bd4b7060a7a3f41ce08f32297ab47547a6b01b"
 python-versions = "^3.7"
 
 [metadata.files]
@@ -215,6 +284,10 @@ beautifulsoup4 = [
     {file = "beautifulsoup4-4.9.1-py3-none-any.whl", hash = "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8"},
     {file = "beautifulsoup4-4.9.1.tar.gz", hash = "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7"},
 ]
+bleach = [
+    {file = "bleach-3.1.5-py2.py3-none-any.whl", hash = "sha256:2bce3d8fab545a6528c8fa5d9f9ae8ebc85a56da365c7f85180bfe96a35ef22f"},
+    {file = "bleach-3.1.5.tar.gz", hash = "sha256:3c4c520fdb9db59ef139915a5db79f8b51bc2a7257ea0389f30c846883430a4b"},
+]
 chardet = [
     {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
     {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"},
@@ -223,10 +296,19 @@ django = [
     {file = "Django-3.0.7-py3-none-any.whl", hash = "sha256:e1630333248c9b3d4e38f02093a26f1e07b271ca896d73097457996e0fae12e8"},
     {file = "Django-3.0.7.tar.gz", hash = "sha256:5052b34b34b3425233c682e0e11d658fd6efd587d11335a0203d827224ada8f2"},
 ]
+django-bleach = [
+    {file = "django-bleach-0.6.1.tar.gz", hash = "sha256:674709c26040618aff0741ce8261fd151e5ead405bd50568c2034662d69daac3"},
+    {file = "django_bleach-0.6.1-py2.py3-none-any.whl", hash = "sha256:59de95cd98f924992313821ab7f94cd64a03aa900ca980bd3b062d8aef1a7954"},
+]
 django-ordered-model = [
     {file = "django-ordered-model-3.4.1.tar.gz", hash = "sha256:d867166ed4dd12501139e119cbbc5b4d19798a3e72740aef0af4879ba97102cf"},
     {file = "django_ordered_model-3.4.1-py3-none-any.whl", hash = "sha256:29af6624cf3505daaf0df00e2df1d0726dd777b95e08f304d5ad0264092aa934"},
 ]
+feedparser = [
+    {file = "feedparser-5.2.1.tar.bz2", hash = "sha256:ce875495c90ebd74b179855449040003a1beb40cd13d5f037a0654251e260b02"},
+    {file = "feedparser-5.2.1.tar.gz", hash = "sha256:bd030652c2d08532c034c27fcd7c85868e7fa3cb2b17f230a44a6bbc92519bf9"},
+    {file = "feedparser-5.2.1.zip", hash = "sha256:cd2485472e41471632ed3029d44033ee420ad0b57111db95c240c9160a85831c"},
+]
 humanize = [
     {file = "humanize-2.4.0-py3-none-any.whl", hash = "sha256:07dd1293bac6c77daa5ccdc22c0b41b2315bee0e339a9f035ba86a9f1a272002"},
     {file = "humanize-2.4.0.tar.gz", hash = "sha256:42ae7d54b398c01bd100847f6cb0fc9e381c21be8ad3f8e2929135e48dbff026"},
@@ -254,6 +336,10 @@ multidict = [
     {file = "multidict-4.7.6-cp38-cp38-win_amd64.whl", hash = "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19"},
     {file = "multidict-4.7.6.tar.gz", hash = "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430"},
 ]
+packaging = [
+    {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"},
+    {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"},
+]
 pillow = [
     {file = "Pillow-7.1.2-cp35-cp35m-macosx_10_10_intel.whl", hash = "sha256:ae2b270f9a0b8822b98655cb3a59cdb1bd54a34807c6c56b76dd2e786c3b7db3"},
     {file = "Pillow-7.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:d23e2aa9b969cf9c26edfb4b56307792b8b374202810bd949effd1c6e11ebd6d"},
@@ -292,10 +378,18 @@ psutil = [
     {file = "psutil-5.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26"},
     {file = "psutil-5.7.0.tar.gz", hash = "sha256:685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e"},
 ]
+pyparsing = [
+    {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"},
+    {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"},
+]
 pytz = [
     {file = "pytz-2020.1-py2.py3-none-any.whl", hash = "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed"},
     {file = "pytz-2020.1.tar.gz", hash = "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"},
 ]
+six = [
+    {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"},
+    {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"},
+]
 soupsieve = [
     {file = "soupsieve-1.9.6-py2.py3-none-any.whl", hash = "sha256:feb1e937fa26a69e08436aad4a9037cd7e1d4c7212909502ba30701247ff8abd"},
     {file = "soupsieve-1.9.6.tar.gz", hash = "sha256:7985bacc98c34923a439967c1a602dc4f1e15f923b6fcf02344184f86cc7efaa"},
@@ -304,6 +398,10 @@ sqlparse = [
     {file = "sqlparse-0.3.1-py2.py3-none-any.whl", hash = "sha256:022fb9c87b524d1f7862b3037e541f68597a730a8843245c349fc93e1643dc4e"},
     {file = "sqlparse-0.3.1.tar.gz", hash = "sha256:e162203737712307dfe78860cc56c8da8a852ab2ee33750e33aeadf38d12c548"},
 ]
+webencodings = [
+    {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"},
+    {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"},
+]
 yarl = [
     {file = "yarl-1.4.2-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b"},
     {file = "yarl-1.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1"},

pyproject.toml

@@ -13,6 +13,8 @@ beautifulsoup4 = "^4.9.1"
 psutil = "^5.7.0"
 humanize = "^2.4.0"
 aiohttp = "^3.6.2"
+feedparser = "^5.2.1"
+django-bleach = "^0.6.1"
 
 [tool.poetry.dev-dependencies]
 

sdbs_infra/dashboard/admin.py

@@ -1,7 +1,7 @@
 from django.contrib import admin
 from ordered_model.admin import OrderedModelAdmin
 
-from sdbs_infra.dashboard.models import Service, Link, Machine
+from sdbs_infra.dashboard.models import Service, Link, Machine, Feed
 
 
 class LinkAdmin(OrderedModelAdmin):
@@ -19,3 +19,4 @@ class MachineAdmin(OrderedModelAdmin):
 admin.site.register(Link, LinkAdmin)
 admin.site.register(Service, ServiceAdmin)
 admin.site.register(Machine, MachineAdmin)
+admin.site.register(Feed)

sdbs_infra/dashboard/models.py

@@ -44,3 +44,11 @@ class Machine(OrderedModel):
 
     def __str__(self):
         return f"{self.short_name}"
+
+
+class Feed(models.Model):
+    short_name = models.CharField(null=True, blank=True, max_length=64)
+    url = models.URLField()
+
+    def __str__(self):
+        return f"{self.short_name} ({self.url})" if self.short_name else self.url

sdbs_infra/dashboard/static/main.css

@@ -144,4 +144,31 @@ h2 {
 .stats em {
     font-style: normal;
     font-weight: bold;
+}
+
+.firehose {
+    margin: 0 4rem 2rem 4rem;
+}
+
+.firehose .logs {
+    list-style: none;
+    padding: 0;
+}
+
+.firehose .logs .header .timestamp {
+    white-space: nowrap;
+    color: darkgray;
+    margin-right: 1em;
+}
+
+.firehose .logs .header {
+    display: flex;
+}
+
+.firehose .logs li {
+    margin: .75em 0;
+}
+
+.firehose .logs .summary {
+    margin: .25em 0 0 4rem;
 }

sdbs_infra/dashboard/templates/index.html

@@ -1,4 +1,6 @@
 {% load static %}
+{% load bleach_tags %}
+
 <!DOCTYPE html>
 <html lang="en">
 <head>
@@ -52,6 +54,9 @@
 </section>
 <h2>machines</h2>
 <section class="machines boxes">
+    <section class="stats">
+        VPS STATS &mdash; {{ vps_stats|safe }}
+    </section>
     {% for machine in machines %}
         <a class="machine box status-{{ machine.status }}" href="{{ machine.url|default:"#" }}">
             <section class="box-content">
@@ -71,8 +76,23 @@
         </a>
     {% endfor %}
 </section>
-<section class="stats">
-    VPS STATS &mdash; {{ vps_stats|safe }}
+<h2>logs firehose</h2>
+<section class="firehose">
+    <ul class="logs">
+        {% for feed_item in feed_items %}
+            <li>
+                <div class="header">
+                    <div class="timestamp">[{{ feed_item.published_datetime|date:"Y-m-d | H:i:s" }}]</div>
+                    <div class title="title">{{ feed_item.title }}</div>
+                </div>
+                {% if feed_item.summary %}
+                    <div class="summary">
+                        {{ feed_item.summary|bleach|safe }}
+                    </div>
+                {% endif %}
+            </li>
+        {% endfor %}
+    </ul>
 </section>
 </body>
 </html>

sdbs_infra/dashboard/views.py

@@ -3,9 +3,11 @@ import socket
 import time
 from collections import namedtuple
 from datetime import datetime
+from operator import itemgetter
 from urllib.parse import urlparse
 
 import aiohttp
+import feedparser
 import psutil
 from aiohttp import ClientConnectorError
 from bs4 import BeautifulSoup
@@ -13,7 +15,7 @@ from django.views.generic import TemplateView
 from humanize import naturalsize
 
 from sdbs_infra import settings
-from sdbs_infra.dashboard.models import Service, Status, Link, Machine
+from sdbs_infra.dashboard.models import Service, Status, Link, Machine, Feed
 
 
 class IndexView(TemplateView):
@@ -25,6 +27,7 @@ class IndexView(TemplateView):
             'links': asyncio.run(self.process_links(list(Link.objects.all()))),
             'services': asyncio.run(self.process_services(list(Service.objects.all()))),
             'machines': asyncio.run(self.process_machines(list(Machine.objects.all()))),
+            'feed_items': asyncio.run(self.process_feeds(list(Feed.objects.all()))),
             'vps_stats': self.vps_stats()
         }
 
@@ -122,6 +125,26 @@ class IndexView(TemplateView):
 
         return result
 
+    async def process_feeds(self, feeds):
+        result = []
+
+        session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=5, sock_connect=1))
+
+        for feed in feeds:
+            try:
+                async with session.get(feed.url) as response:
+                    parsed_feed = feedparser.parse(await response.text())
+                    entries = parsed_feed.entries
+                    for entry in entries:
+                        entry.published_datetime = datetime(*entry.published_parsed[0:6])
+                    result.extend(parsed_feed.entries)
+            except (asyncio.TimeoutError, ClientConnectorError):
+                continue
+
+        result.sort(key=itemgetter('published_parsed'), reverse=True)
+
+        return result
+
     @staticmethod
     def extract_favicon(url, index_text):
         if not index_text:

sdbs_infra/settings.py

@@ -39,7 +39,8 @@ INSTALLED_APPS = [
     'django.contrib.sessions',
     'django.contrib.staticfiles',
     'ordered_model',
-    'sdbs_infra.dashboard'
+    'sdbs_infra.dashboard',
+    'django_bleach'
 ]
 
 MIDDLEWARE = [
@@ -129,4 +130,6 @@ STATIC_ROOT = os.path.join(BASE_DIR, "static")
 MEDIA_URL = '/media/'
 MEDIA_ROOT = os.path.join(BASE_DIR, "media")
 
+BLEACH_STRIP_TAGS = True
+
 HEALTCHECKS_API_KEY = os.getenv("HEALTHCHECKS_API_KEY")