diff --git a/poetry.lock b/poetry.lock index 96c054f..52cc9d4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -64,6 +64,19 @@ soupsieve = [">1.2", "<2.0"] html5lib = ["html5lib"] lxml = ["lxml"] +[[package]] +category = "main" +description = "An easy safelist-based HTML-sanitizing tool." +name = "bleach" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" +version = "3.1.5" + +[package.dependencies] +packaging = "*" +six = ">=1.9.0" +webencodings = "*" + [[package]] category = "main" description = "Universal encoding detector for Python 2 and 3" @@ -89,6 +102,18 @@ sqlparse = ">=0.2.2" argon2 = ["argon2-cffi (>=16.1.0)"] bcrypt = ["bcrypt"] +[[package]] +category = "main" +description = "Easily use bleach with Django models and templates" +name = "django-bleach" +optional = false +python-versions = "*" +version = "0.6.1" + +[package.dependencies] +Django = ">=1.11" +bleach = ">=1.5.0" + [[package]] category = "main" description = "Allows Django models to be ordered and provides a simple admin interface for reordering them." @@ -97,6 +122,14 @@ optional = false python-versions = "*" version = "3.4.1" +[[package]] +category = "main" +description = "Universal feed parser, handles RSS 0.9x, RSS 1.0, RSS 2.0, CDF, Atom 0.3, and Atom 1.0 feeds" +name = "feedparser" +optional = false +python-versions = "*" +version = "5.2.1" + [[package]] category = "main" description = "Python humanize utilities" @@ -124,6 +157,18 @@ optional = false python-versions = ">=3.5" version = "4.7.6" +[[package]] +category = "main" +description = "Core utilities for Python packages" +name = "packaging" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "20.4" + +[package.dependencies] +pyparsing = ">=2.0.2" +six = "*" + [[package]] category = "main" description = "Python Imaging Library (Fork)" @@ -143,6 +188,14 @@ version = "5.7.0" [package.extras] enum = ["enum34"] +[[package]] +category = "main" +description = "Python parsing module" +name = "pyparsing" +optional = false +python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +version = "2.4.7" + [[package]] category = "main" description = "World timezone definitions, modern and historical" @@ -151,6 +204,14 @@ optional = false python-versions = "*" version = "2020.1" +[[package]] +category = "main" +description = "Python 2 and 3 compatibility utilities" +name = "six" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +version = "1.15.0" + [[package]] category = "main" description = "A modern CSS selector implementation for Beautiful Soup." @@ -167,6 +228,14 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "0.3.1" +[[package]] +category = "main" +description = "Character encoding aliases for legacy web content" +name = "webencodings" +optional = false +python-versions = "*" +version = "0.5.1" + [[package]] category = "main" description = "Yet another URL library" @@ -180,7 +249,7 @@ idna = ">=2.0" multidict = ">=4.0" [metadata] -content-hash = "c17c9f2b361876b298b7a7a2917926979b5681ee7d037fbb2c19bfd8f5f3f352" +content-hash = "5fcbd04cee09c4e0ec32a23ab7bd4b7060a7a3f41ce08f32297ab47547a6b01b" python-versions = "^3.7" [metadata.files] @@ -215,6 +284,10 @@ beautifulsoup4 = [ {file = "beautifulsoup4-4.9.1-py3-none-any.whl", hash = "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8"}, {file = "beautifulsoup4-4.9.1.tar.gz", hash = "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7"}, ] +bleach = [ + {file = "bleach-3.1.5-py2.py3-none-any.whl", hash = "sha256:2bce3d8fab545a6528c8fa5d9f9ae8ebc85a56da365c7f85180bfe96a35ef22f"}, + {file = "bleach-3.1.5.tar.gz", hash = "sha256:3c4c520fdb9db59ef139915a5db79f8b51bc2a7257ea0389f30c846883430a4b"}, +] chardet = [ {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, @@ -223,10 +296,19 @@ django = [ {file = "Django-3.0.7-py3-none-any.whl", hash = "sha256:e1630333248c9b3d4e38f02093a26f1e07b271ca896d73097457996e0fae12e8"}, {file = "Django-3.0.7.tar.gz", hash = "sha256:5052b34b34b3425233c682e0e11d658fd6efd587d11335a0203d827224ada8f2"}, ] +django-bleach = [ + {file = "django-bleach-0.6.1.tar.gz", hash = "sha256:674709c26040618aff0741ce8261fd151e5ead405bd50568c2034662d69daac3"}, + {file = "django_bleach-0.6.1-py2.py3-none-any.whl", hash = "sha256:59de95cd98f924992313821ab7f94cd64a03aa900ca980bd3b062d8aef1a7954"}, +] django-ordered-model = [ {file = "django-ordered-model-3.4.1.tar.gz", hash = "sha256:d867166ed4dd12501139e119cbbc5b4d19798a3e72740aef0af4879ba97102cf"}, {file = "django_ordered_model-3.4.1-py3-none-any.whl", hash = "sha256:29af6624cf3505daaf0df00e2df1d0726dd777b95e08f304d5ad0264092aa934"}, ] +feedparser = [ + {file = "feedparser-5.2.1.tar.bz2", hash = "sha256:ce875495c90ebd74b179855449040003a1beb40cd13d5f037a0654251e260b02"}, + {file = "feedparser-5.2.1.tar.gz", hash = "sha256:bd030652c2d08532c034c27fcd7c85868e7fa3cb2b17f230a44a6bbc92519bf9"}, + {file = "feedparser-5.2.1.zip", hash = "sha256:cd2485472e41471632ed3029d44033ee420ad0b57111db95c240c9160a85831c"}, +] humanize = [ {file = "humanize-2.4.0-py3-none-any.whl", hash = "sha256:07dd1293bac6c77daa5ccdc22c0b41b2315bee0e339a9f035ba86a9f1a272002"}, {file = "humanize-2.4.0.tar.gz", hash = "sha256:42ae7d54b398c01bd100847f6cb0fc9e381c21be8ad3f8e2929135e48dbff026"}, @@ -254,6 +336,10 @@ multidict = [ {file = "multidict-4.7.6-cp38-cp38-win_amd64.whl", hash = "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19"}, {file = "multidict-4.7.6.tar.gz", hash = "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430"}, ] +packaging = [ + {file = "packaging-20.4-py2.py3-none-any.whl", hash = "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181"}, + {file = "packaging-20.4.tar.gz", hash = "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8"}, +] pillow = [ {file = "Pillow-7.1.2-cp35-cp35m-macosx_10_10_intel.whl", hash = "sha256:ae2b270f9a0b8822b98655cb3a59cdb1bd54a34807c6c56b76dd2e786c3b7db3"}, {file = "Pillow-7.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:d23e2aa9b969cf9c26edfb4b56307792b8b374202810bd949effd1c6e11ebd6d"}, @@ -292,10 +378,18 @@ psutil = [ {file = "psutil-5.7.0-cp38-cp38-win_amd64.whl", hash = "sha256:d84029b190c8a66a946e28b4d3934d2ca1528ec94764b180f7d6ea57b0e75e26"}, {file = "psutil-5.7.0.tar.gz", hash = "sha256:685ec16ca14d079455892f25bd124df26ff9137664af445563c1bd36629b5e0e"}, ] +pyparsing = [ + {file = "pyparsing-2.4.7-py2.py3-none-any.whl", hash = "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"}, + {file = "pyparsing-2.4.7.tar.gz", hash = "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1"}, +] pytz = [ {file = "pytz-2020.1-py2.py3-none-any.whl", hash = "sha256:a494d53b6d39c3c6e44c3bec237336e14305e4f29bbf800b599253057fbb79ed"}, {file = "pytz-2020.1.tar.gz", hash = "sha256:c35965d010ce31b23eeb663ed3cc8c906275d6be1a34393a1d73a41febf4a048"}, ] +six = [ + {file = "six-1.15.0-py2.py3-none-any.whl", hash = "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"}, + {file = "six-1.15.0.tar.gz", hash = "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259"}, +] soupsieve = [ {file = "soupsieve-1.9.6-py2.py3-none-any.whl", hash = "sha256:feb1e937fa26a69e08436aad4a9037cd7e1d4c7212909502ba30701247ff8abd"}, {file = "soupsieve-1.9.6.tar.gz", hash = "sha256:7985bacc98c34923a439967c1a602dc4f1e15f923b6fcf02344184f86cc7efaa"}, @@ -304,6 +398,10 @@ sqlparse = [ {file = "sqlparse-0.3.1-py2.py3-none-any.whl", hash = "sha256:022fb9c87b524d1f7862b3037e541f68597a730a8843245c349fc93e1643dc4e"}, {file = "sqlparse-0.3.1.tar.gz", hash = "sha256:e162203737712307dfe78860cc56c8da8a852ab2ee33750e33aeadf38d12c548"}, ] +webencodings = [ + {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, + {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, +] yarl = [ {file = "yarl-1.4.2-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b"}, {file = "yarl-1.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1"}, diff --git a/pyproject.toml b/pyproject.toml index 57f16c7..931782d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,6 +13,8 @@ beautifulsoup4 = "^4.9.1" psutil = "^5.7.0" humanize = "^2.4.0" aiohttp = "^3.6.2" +feedparser = "^5.2.1" +django-bleach = "^0.6.1" [tool.poetry.dev-dependencies] diff --git a/sdbs_infra/dashboard/admin.py b/sdbs_infra/dashboard/admin.py index 1921c42..e133593 100644 --- a/sdbs_infra/dashboard/admin.py +++ b/sdbs_infra/dashboard/admin.py @@ -1,7 +1,7 @@ from django.contrib import admin from ordered_model.admin import OrderedModelAdmin -from sdbs_infra.dashboard.models import Service, Link, Machine +from sdbs_infra.dashboard.models import Service, Link, Machine, Feed class LinkAdmin(OrderedModelAdmin): @@ -19,3 +19,4 @@ class MachineAdmin(OrderedModelAdmin): admin.site.register(Link, LinkAdmin) admin.site.register(Service, ServiceAdmin) admin.site.register(Machine, MachineAdmin) +admin.site.register(Feed) diff --git a/sdbs_infra/dashboard/models.py b/sdbs_infra/dashboard/models.py index 179ddf5..6ba1852 100644 --- a/sdbs_infra/dashboard/models.py +++ b/sdbs_infra/dashboard/models.py @@ -44,3 +44,11 @@ class Machine(OrderedModel): def __str__(self): return f"{self.short_name}" + + +class Feed(models.Model): + short_name = models.CharField(null=True, blank=True, max_length=64) + url = models.URLField() + + def __str__(self): + return f"{self.short_name} ({self.url})" if self.short_name else self.url diff --git a/sdbs_infra/dashboard/static/main.css b/sdbs_infra/dashboard/static/main.css index c434368..2c2115e 100644 --- a/sdbs_infra/dashboard/static/main.css +++ b/sdbs_infra/dashboard/static/main.css @@ -144,4 +144,31 @@ h2 { .stats em { font-style: normal; font-weight: bold; +} + +.firehose { + margin: 0 4rem 2rem 4rem; +} + +.firehose .logs { + list-style: none; + padding: 0; +} + +.firehose .logs .header .timestamp { + white-space: nowrap; + color: darkgray; + margin-right: 1em; +} + +.firehose .logs .header { + display: flex; +} + +.firehose .logs li { + margin: .75em 0; +} + +.firehose .logs .summary { + margin: .25em 0 0 4rem; } \ No newline at end of file diff --git a/sdbs_infra/dashboard/templates/index.html b/sdbs_infra/dashboard/templates/index.html index dd3c87d..5a131e4 100644 --- a/sdbs_infra/dashboard/templates/index.html +++ b/sdbs_infra/dashboard/templates/index.html @@ -1,4 +1,6 @@ {% load static %} +{% load bleach_tags %} + @@ -52,6 +54,9 @@

machines

+
+ VPS STATS — {{ vps_stats|safe }} +
{% for machine in machines %}
@@ -71,8 +76,23 @@ {% endfor %}
-
- VPS STATS — {{ vps_stats|safe }} +

logs firehose

+
+
    + {% for feed_item in feed_items %} +
  • +
    +
    [{{ feed_item.published_datetime|date:"Y-m-d | H:i:s" }}]
    +
    {{ feed_item.title }}
    +
    + {% if feed_item.summary %} +
    + {{ feed_item.summary|bleach|safe }} +
    + {% endif %} +
    • + {% endfor %} +
\ No newline at end of file diff --git a/sdbs_infra/dashboard/views.py b/sdbs_infra/dashboard/views.py index a918624..2cd1bd7 100644 --- a/sdbs_infra/dashboard/views.py +++ b/sdbs_infra/dashboard/views.py @@ -3,9 +3,11 @@ import socket import time from collections import namedtuple from datetime import datetime +from operator import itemgetter from urllib.parse import urlparse import aiohttp +import feedparser import psutil from aiohttp import ClientConnectorError from bs4 import BeautifulSoup @@ -13,7 +15,7 @@ from django.views.generic import TemplateView from humanize import naturalsize from sdbs_infra import settings -from sdbs_infra.dashboard.models import Service, Status, Link, Machine +from sdbs_infra.dashboard.models import Service, Status, Link, Machine, Feed class IndexView(TemplateView): @@ -25,6 +27,7 @@ class IndexView(TemplateView): 'links': asyncio.run(self.process_links(list(Link.objects.all()))), 'services': asyncio.run(self.process_services(list(Service.objects.all()))), 'machines': asyncio.run(self.process_machines(list(Machine.objects.all()))), + 'feed_items': asyncio.run(self.process_feeds(list(Feed.objects.all()))), 'vps_stats': self.vps_stats() } @@ -122,6 +125,26 @@ class IndexView(TemplateView): return result + async def process_feeds(self, feeds): + result = [] + + session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=5, sock_connect=1)) + + for feed in feeds: + try: + async with session.get(feed.url) as response: + parsed_feed = feedparser.parse(await response.text()) + entries = parsed_feed.entries + for entry in entries: + entry.published_datetime = datetime(*entry.published_parsed[0:6]) + result.extend(parsed_feed.entries) + except (asyncio.TimeoutError, ClientConnectorError): + continue + + result.sort(key=itemgetter('published_parsed'), reverse=True) + + return result + @staticmethod def extract_favicon(url, index_text): if not index_text: diff --git a/sdbs_infra/settings.py b/sdbs_infra/settings.py index d0f742c..09c6de0 100644 --- a/sdbs_infra/settings.py +++ b/sdbs_infra/settings.py @@ -39,7 +39,8 @@ INSTALLED_APPS = [ 'django.contrib.sessions', 'django.contrib.staticfiles', 'ordered_model', - 'sdbs_infra.dashboard' + 'sdbs_infra.dashboard', + 'django_bleach' ] MIDDLEWARE = [ @@ -129,4 +130,6 @@ STATIC_ROOT = os.path.join(BASE_DIR, "static") MEDIA_URL = '/media/' MEDIA_ROOT = os.path.join(BASE_DIR, "media") +BLEACH_STRIP_TAGS = True + HEALTCHECKS_API_KEY = os.getenv("HEALTHCHECKS_API_KEY")