From eba01c013a6358edcb1f0e7c101be8d20f6c0303 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Ml=C3=A1dek?= Date: Tue, 16 Jun 2020 22:20:26 +0200 Subject: [PATCH] implement asyncio; unknown status for timeouts --- poetry.lock | 146 +++++++++++++++++++++- pyproject.toml | 1 + sdbs_infra/dashboard/models.py | 47 +------ sdbs_infra/dashboard/static/main.css | 4 + sdbs_infra/dashboard/templates/index.html | 8 +- sdbs_infra/dashboard/views.py | 61 ++++++++- 6 files changed, 214 insertions(+), 53 deletions(-) diff --git a/poetry.lock b/poetry.lock index 8a32ba1..96c054f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,3 +1,21 @@ +[[package]] +category = "main" +description = "Async http client/server framework (asyncio)" +name = "aiohttp" +optional = false +python-versions = ">=3.5.3" +version = "3.6.2" + +[package.dependencies] +async-timeout = ">=3.0,<4.0" +attrs = ">=17.3.0" +chardet = ">=2.0,<4.0" +multidict = ">=4.5,<5.0" +yarl = ">=1.0,<2.0" + +[package.extras] +speedups = ["aiodns", "brotlipy", "cchardet"] + [[package]] category = "main" description = "ASGI specs, helper code, and adapters" @@ -9,6 +27,28 @@ version = "3.2.7" [package.extras] tests = ["pytest (>=4.3.0,<4.4.0)", "pytest-asyncio (>=0.10.0,<0.11.0)"] +[[package]] +category = "main" +description = "Timeout context manager for asyncio programs" +name = "async-timeout" +optional = false +python-versions = ">=3.5.3" +version = "3.0.1" + +[[package]] +category = "main" +description = "Classes Without Boilerplate" +name = "attrs" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "19.3.0" + +[package.extras] +azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"] +dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"] +docs = ["sphinx", "zope.interface"] +tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"] + [[package]] category = "main" description = "Screen-scraping library" @@ -24,6 +64,14 @@ soupsieve = [">1.2", "<2.0"] html5lib = ["html5lib"] lxml = ["lxml"] +[[package]] +category = "main" +description = "Universal encoding detector for Python 2 and 3" +name = "chardet" +optional = false +python-versions = "*" +version = "3.0.4" + [[package]] category = "main" description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design." @@ -60,6 +108,22 @@ version = "2.4.0" [package.extras] tests = ["freezegun", "pytest", "pytest-cov"] +[[package]] +category = "main" +description = "Internationalized Domain Names in Applications (IDNA)" +name = "idna" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +version = "2.9" + +[[package]] +category = "main" +description = "multidict implementation" +name = "multidict" +optional = false +python-versions = ">=3.5" +version = "4.7.6" + [[package]] category = "main" description = "Python Imaging Library (Fork)" @@ -103,20 +167,58 @@ optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" version = "0.3.1" +[[package]] +category = "main" +description = "Yet another URL library" +name = "yarl" +optional = false +python-versions = ">=3.5" +version = "1.4.2" + +[package.dependencies] +idna = ">=2.0" +multidict = ">=4.0" + [metadata] -content-hash = "c2be494386f51759b009fd7ed3aead33cb56578ddf36c2102d7c3bb10e7bf056" +content-hash = "c17c9f2b361876b298b7a7a2917926979b5681ee7d037fbb2c19bfd8f5f3f352" python-versions = "^3.7" [metadata.files] +aiohttp = [ + {file = "aiohttp-3.6.2-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e"}, + {file = "aiohttp-3.6.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec"}, + {file = "aiohttp-3.6.2-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48"}, + {file = "aiohttp-3.6.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59"}, + {file = "aiohttp-3.6.2-cp36-cp36m-win32.whl", hash = "sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a"}, + {file = "aiohttp-3.6.2-cp36-cp36m-win_amd64.whl", hash = "sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17"}, + {file = "aiohttp-3.6.2-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a"}, + {file = "aiohttp-3.6.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd"}, + {file = "aiohttp-3.6.2-cp37-cp37m-win32.whl", hash = "sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"}, + {file = "aiohttp-3.6.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654"}, + {file = "aiohttp-3.6.2-py3-none-any.whl", hash = "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4"}, + {file = "aiohttp-3.6.2.tar.gz", hash = "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326"}, +] asgiref = [ {file = "asgiref-3.2.7-py2.py3-none-any.whl", hash = "sha256:9ca8b952a0a9afa61d30aa6d3d9b570bb3fd6bafcf7ec9e6bed43b936133db1c"}, {file = "asgiref-3.2.7.tar.gz", hash = "sha256:8036f90603c54e93521e5777b2b9a39ba1bad05773fcf2d208f0299d1df58ce5"}, ] +async-timeout = [ + {file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"}, + {file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"}, +] +attrs = [ + {file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"}, + {file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"}, +] beautifulsoup4 = [ {file = "beautifulsoup4-4.9.1-py2-none-any.whl", hash = "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c"}, {file = "beautifulsoup4-4.9.1-py3-none-any.whl", hash = "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8"}, {file = "beautifulsoup4-4.9.1.tar.gz", hash = "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7"}, ] +chardet = [ + {file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"}, + {file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"}, +] django = [ {file = "Django-3.0.7-py3-none-any.whl", hash = "sha256:e1630333248c9b3d4e38f02093a26f1e07b271ca896d73097457996e0fae12e8"}, {file = "Django-3.0.7.tar.gz", hash = "sha256:5052b34b34b3425233c682e0e11d658fd6efd587d11335a0203d827224ada8f2"}, @@ -129,6 +231,29 @@ humanize = [ {file = "humanize-2.4.0-py3-none-any.whl", hash = "sha256:07dd1293bac6c77daa5ccdc22c0b41b2315bee0e339a9f035ba86a9f1a272002"}, {file = "humanize-2.4.0.tar.gz", hash = "sha256:42ae7d54b398c01bd100847f6cb0fc9e381c21be8ad3f8e2929135e48dbff026"}, ] +idna = [ + {file = "idna-2.9-py2.py3-none-any.whl", hash = "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"}, + {file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"}, +] +multidict = [ + {file = "multidict-4.7.6-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000"}, + {file = "multidict-4.7.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a"}, + {file = "multidict-4.7.6-cp35-cp35m-win32.whl", hash = "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5"}, + {file = "multidict-4.7.6-cp35-cp35m-win_amd64.whl", hash = "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3"}, + {file = "multidict-4.7.6-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87"}, + {file = "multidict-4.7.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2"}, + {file = "multidict-4.7.6-cp36-cp36m-win32.whl", hash = "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7"}, + {file = "multidict-4.7.6-cp36-cp36m-win_amd64.whl", hash = "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463"}, + {file = "multidict-4.7.6-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d"}, + {file = "multidict-4.7.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255"}, + {file = "multidict-4.7.6-cp37-cp37m-win32.whl", hash = "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507"}, + {file = "multidict-4.7.6-cp37-cp37m-win_amd64.whl", hash = "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c"}, + {file = "multidict-4.7.6-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b"}, + {file = "multidict-4.7.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7"}, + {file = "multidict-4.7.6-cp38-cp38-win32.whl", hash = "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d"}, + {file = "multidict-4.7.6-cp38-cp38-win_amd64.whl", hash = "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19"}, + {file = "multidict-4.7.6.tar.gz", hash = "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430"}, +] pillow = [ {file = "Pillow-7.1.2-cp35-cp35m-macosx_10_10_intel.whl", hash = "sha256:ae2b270f9a0b8822b98655cb3a59cdb1bd54a34807c6c56b76dd2e786c3b7db3"}, {file = "Pillow-7.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:d23e2aa9b969cf9c26edfb4b56307792b8b374202810bd949effd1c6e11ebd6d"}, @@ -179,3 +304,22 @@ sqlparse = [ {file = "sqlparse-0.3.1-py2.py3-none-any.whl", hash = "sha256:022fb9c87b524d1f7862b3037e541f68597a730a8843245c349fc93e1643dc4e"}, {file = "sqlparse-0.3.1.tar.gz", hash = "sha256:e162203737712307dfe78860cc56c8da8a852ab2ee33750e33aeadf38d12c548"}, ] +yarl = [ + {file = "yarl-1.4.2-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b"}, + {file = "yarl-1.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1"}, + {file = "yarl-1.4.2-cp35-cp35m-win32.whl", hash = "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080"}, + {file = "yarl-1.4.2-cp35-cp35m-win_amd64.whl", hash = "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a"}, + {file = "yarl-1.4.2-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f"}, + {file = "yarl-1.4.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea"}, + {file = "yarl-1.4.2-cp36-cp36m-win32.whl", hash = "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb"}, + {file = "yarl-1.4.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70"}, + {file = "yarl-1.4.2-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d"}, + {file = "yarl-1.4.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce"}, + {file = "yarl-1.4.2-cp37-cp37m-win32.whl", hash = "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2"}, + {file = "yarl-1.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce"}, + {file = "yarl-1.4.2-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b"}, + {file = "yarl-1.4.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae"}, + {file = "yarl-1.4.2-cp38-cp38-win32.whl", hash = "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462"}, + {file = "yarl-1.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6"}, + {file = "yarl-1.4.2.tar.gz", hash = "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b"}, +] diff --git a/pyproject.toml b/pyproject.toml index a0161a9..57f16c7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,6 +12,7 @@ Pillow = "^7.1.2" beautifulsoup4 = "^4.9.1" psutil = "^5.7.0" humanize = "^2.4.0" +aiohttp = "^3.6.2" [tool.poetry.dev-dependencies] diff --git a/sdbs_infra/dashboard/models.py b/sdbs_infra/dashboard/models.py index e93100b..b505434 100644 --- a/sdbs_infra/dashboard/models.py +++ b/sdbs_infra/dashboard/models.py @@ -1,13 +1,8 @@ import socket -import urllib.request from enum import Enum -from socket import timeout -from urllib.error import URLError -from urllib.request import Request from bs4 import BeautifulSoup from django.db import models -from django.utils.functional import cached_property from ordered_model.models import OrderedModel @@ -25,44 +20,4 @@ class Service(OrderedModel): url = models.URLField() def __str__(self): - return f"{self.short_name} ({self.url})" - - @cached_property - def index_request(self): - try: - request = Request(self.url, headers={'User-Agent': 'its just me humble status page'}) - return urllib.request.urlopen(request, timeout=1) - except (URLError, timeout): - return None - - def get_status(self): - if self.port: - a_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - location = ("localhost", self.port) - result_of_check = a_socket.connect_ex(location) - if result_of_check == 0: - return ServiceStatus.OK - else: - return ServiceStatus.DOWN - - if self.index_request and self.index_request.getcode() == 200: - return ServiceStatus.OK - else: - return ServiceStatus.DOWN - - @cached_property - def image_or_favicon(self): - if self.image: - return self.image.url - - if self.index_request: - parsed_html = BeautifulSoup(self.index_request, features="html.parser") - link_tags = parsed_html.find_all('link') - for rel in ['apple-touch-icon', 'shortcut', 'icon']: - for link_tag in link_tags: - if rel in link_tag.attrs['rel']: - result = link_tag.attrs['href'] - if self.url not in result: - return self.url + (result if result.startswith("/") else f"/{result}") - else: - return result + return f"{self.short_name} ({self.url})" \ No newline at end of file diff --git a/sdbs_infra/dashboard/static/main.css b/sdbs_infra/dashboard/static/main.css index e86d8cf..ff62529 100644 --- a/sdbs_infra/dashboard/static/main.css +++ b/sdbs_infra/dashboard/static/main.css @@ -104,6 +104,10 @@ main { color: darkred; } +.status-unknown .service-status { + color: darkorange; +} + .stats { text-align: justify; padding: 1.5rem 0; diff --git a/sdbs_infra/dashboard/templates/index.html b/sdbs_infra/dashboard/templates/index.html index 3da3981..08b5a46 100644 --- a/sdbs_infra/dashboard/templates/index.html +++ b/sdbs_infra/dashboard/templates/index.html @@ -14,10 +14,10 @@
{% for service in services %} - +
- {% if service.image_or_favicon %} - image for {{ service }} + {% if service.image_url %} + image for {{ service.short_name }} {% endif %}

{{ service.short_name }}

@@ -27,7 +27,7 @@
- STATUS: {{ service.get_status.value }} + STATUS: {{ service.status }}
{% endfor %} diff --git a/sdbs_infra/dashboard/views.py b/sdbs_infra/dashboard/views.py index b951ffd..c996234 100644 --- a/sdbs_infra/dashboard/views.py +++ b/sdbs_infra/dashboard/views.py @@ -1,11 +1,15 @@ +import asyncio +import socket import time from collections import namedtuple +import aiohttp import psutil +from bs4 import BeautifulSoup from django.views.generic import TemplateView from humanize import naturalsize -from sdbs_infra.dashboard.models import Service +from sdbs_infra.dashboard.models import Service, ServiceStatus class IndexView(TemplateView): @@ -13,10 +17,63 @@ class IndexView(TemplateView): def get_context_data(self, **kwargs): return { - 'services': Service.objects.all(), + 'services': asyncio.run(self.process_services(list(Service.objects.all()))), 'vps_stats': self.vps_stats() } + @staticmethod + async def process_services(services): + result = [] + + session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=5, sock_connect=1)) + + for service in services: + index_status, index_text = None, None + if not service.port or not service.image: + try: + async with session.get(service.url) as response: + index_status, index_text = response.status, await response.text() + except asyncio.TimeoutError: + pass + + if service.port: + a_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + location = ("localhost", service.port) + result_of_check = a_socket.connect_ex(location) + if result_of_check == 0: + status = ServiceStatus.OK + else: + status = ServiceStatus.DOWN + elif index_status: + status = ServiceStatus.OK if index_status == 200 else ServiceStatus.DOWN + else: + status = ServiceStatus.UNKNOWN + + image = None + if service.image: + image = service.image.url + elif index_text: + parsed_html = BeautifulSoup(index_text, features="html.parser") + link_tags = parsed_html.find_all('link') + for rel in ['apple-touch-icon', 'shortcut', 'icon']: + for link_tag in link_tags: + if rel in link_tag.attrs['rel']: + link = link_tag.attrs['href'] + if service.url not in link: + image = service.url + (link if link.startswith("/") else f"/{link}") + else: + image = link + + result.append({ + 'status': status.value, + 'image_url': image, + **vars(service) + }) + + await session.close() + + return result + # noinspection PyListCreation @staticmethod def vps_stats():