implement asyncio; unknown status for timeouts

This commit is contained in:
Tomáš Mládek 2020-06-16 22:20:26 +02:00
parent e667f7043e
commit eba01c013a
6 changed files with 214 additions and 53 deletions

146
poetry.lock generated
View file

@ -1,3 +1,21 @@
[[package]]
category = "main"
description = "Async http client/server framework (asyncio)"
name = "aiohttp"
optional = false
python-versions = ">=3.5.3"
version = "3.6.2"
[package.dependencies]
async-timeout = ">=3.0,<4.0"
attrs = ">=17.3.0"
chardet = ">=2.0,<4.0"
multidict = ">=4.5,<5.0"
yarl = ">=1.0,<2.0"
[package.extras]
speedups = ["aiodns", "brotlipy", "cchardet"]
[[package]]
category = "main"
description = "ASGI specs, helper code, and adapters"
@ -9,6 +27,28 @@ version = "3.2.7"
[package.extras]
tests = ["pytest (>=4.3.0,<4.4.0)", "pytest-asyncio (>=0.10.0,<0.11.0)"]
[[package]]
category = "main"
description = "Timeout context manager for asyncio programs"
name = "async-timeout"
optional = false
python-versions = ">=3.5.3"
version = "3.0.1"
[[package]]
category = "main"
description = "Classes Without Boilerplate"
name = "attrs"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "19.3.0"
[package.extras]
azure-pipelines = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "pytest-azurepipelines"]
dev = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface", "sphinx", "pre-commit"]
docs = ["sphinx", "zope.interface"]
tests = ["coverage", "hypothesis", "pympler", "pytest (>=4.3.0)", "six", "zope.interface"]
[[package]]
category = "main"
description = "Screen-scraping library"
@ -24,6 +64,14 @@ soupsieve = [">1.2", "<2.0"]
html5lib = ["html5lib"]
lxml = ["lxml"]
[[package]]
category = "main"
description = "Universal encoding detector for Python 2 and 3"
name = "chardet"
optional = false
python-versions = "*"
version = "3.0.4"
[[package]]
category = "main"
description = "A high-level Python Web framework that encourages rapid development and clean, pragmatic design."
@ -60,6 +108,22 @@ version = "2.4.0"
[package.extras]
tests = ["freezegun", "pytest", "pytest-cov"]
[[package]]
category = "main"
description = "Internationalized Domain Names in Applications (IDNA)"
name = "idna"
optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "2.9"
[[package]]
category = "main"
description = "multidict implementation"
name = "multidict"
optional = false
python-versions = ">=3.5"
version = "4.7.6"
[[package]]
category = "main"
description = "Python Imaging Library (Fork)"
@ -103,20 +167,58 @@ optional = false
python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*"
version = "0.3.1"
[[package]]
category = "main"
description = "Yet another URL library"
name = "yarl"
optional = false
python-versions = ">=3.5"
version = "1.4.2"
[package.dependencies]
idna = ">=2.0"
multidict = ">=4.0"
[metadata]
content-hash = "c2be494386f51759b009fd7ed3aead33cb56578ddf36c2102d7c3bb10e7bf056"
content-hash = "c17c9f2b361876b298b7a7a2917926979b5681ee7d037fbb2c19bfd8f5f3f352"
python-versions = "^3.7"
[metadata.files]
aiohttp = [
{file = "aiohttp-3.6.2-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:1e984191d1ec186881ffaed4581092ba04f7c61582a177b187d3a2f07ed9719e"},
{file = "aiohttp-3.6.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:50aaad128e6ac62e7bf7bd1f0c0a24bc968a0c0590a726d5a955af193544bcec"},
{file = "aiohttp-3.6.2-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:65f31b622af739a802ca6fd1a3076fd0ae523f8485c52924a89561ba10c49b48"},
{file = "aiohttp-3.6.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:ae55bac364c405caa23a4f2d6cfecc6a0daada500274ffca4a9230e7129eac59"},
{file = "aiohttp-3.6.2-cp36-cp36m-win32.whl", hash = "sha256:344c780466b73095a72c616fac5ea9c4665add7fc129f285fbdbca3cccf4612a"},
{file = "aiohttp-3.6.2-cp36-cp36m-win_amd64.whl", hash = "sha256:4c6efd824d44ae697814a2a85604d8e992b875462c6655da161ff18fd4f29f17"},
{file = "aiohttp-3.6.2-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:2f4d1a4fdce595c947162333353d4a44952a724fba9ca3205a3df99a33d1307a"},
{file = "aiohttp-3.6.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:6206a135d072f88da3e71cc501c59d5abffa9d0bb43269a6dcd28d66bfafdbdd"},
{file = "aiohttp-3.6.2-cp37-cp37m-win32.whl", hash = "sha256:b778ce0c909a2653741cb4b1ac7015b5c130ab9c897611df43ae6a58523cb965"},
{file = "aiohttp-3.6.2-cp37-cp37m-win_amd64.whl", hash = "sha256:32e5f3b7e511aa850829fbe5aa32eb455e5534eaa4b1ce93231d00e2f76e5654"},
{file = "aiohttp-3.6.2-py3-none-any.whl", hash = "sha256:460bd4237d2dbecc3b5ed57e122992f60188afe46e7319116da5eb8a9dfedba4"},
{file = "aiohttp-3.6.2.tar.gz", hash = "sha256:259ab809ff0727d0e834ac5e8a283dc5e3e0ecc30c4d80b3cd17a4139ce1f326"},
]
asgiref = [
{file = "asgiref-3.2.7-py2.py3-none-any.whl", hash = "sha256:9ca8b952a0a9afa61d30aa6d3d9b570bb3fd6bafcf7ec9e6bed43b936133db1c"},
{file = "asgiref-3.2.7.tar.gz", hash = "sha256:8036f90603c54e93521e5777b2b9a39ba1bad05773fcf2d208f0299d1df58ce5"},
]
async-timeout = [
{file = "async-timeout-3.0.1.tar.gz", hash = "sha256:0c3c816a028d47f659d6ff5c745cb2acf1f966da1fe5c19c77a70282b25f4c5f"},
{file = "async_timeout-3.0.1-py3-none-any.whl", hash = "sha256:4291ca197d287d274d0b6cb5d6f8f8f82d434ed288f962539ff18cc9012f9ea3"},
]
attrs = [
{file = "attrs-19.3.0-py2.py3-none-any.whl", hash = "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c"},
{file = "attrs-19.3.0.tar.gz", hash = "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72"},
]
beautifulsoup4 = [
{file = "beautifulsoup4-4.9.1-py2-none-any.whl", hash = "sha256:e718f2342e2e099b640a34ab782407b7b676f47ee272d6739e60b8ea23829f2c"},
{file = "beautifulsoup4-4.9.1-py3-none-any.whl", hash = "sha256:a6237df3c32ccfaee4fd201c8f5f9d9df619b93121d01353a64a73ce8c6ef9a8"},
{file = "beautifulsoup4-4.9.1.tar.gz", hash = "sha256:73cc4d115b96f79c7d77c1c7f7a0a8d4c57860d1041df407dd1aae7f07a77fd7"},
]
chardet = [
{file = "chardet-3.0.4-py2.py3-none-any.whl", hash = "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691"},
{file = "chardet-3.0.4.tar.gz", hash = "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae"},
]
django = [
{file = "Django-3.0.7-py3-none-any.whl", hash = "sha256:e1630333248c9b3d4e38f02093a26f1e07b271ca896d73097457996e0fae12e8"},
{file = "Django-3.0.7.tar.gz", hash = "sha256:5052b34b34b3425233c682e0e11d658fd6efd587d11335a0203d827224ada8f2"},
@ -129,6 +231,29 @@ humanize = [
{file = "humanize-2.4.0-py3-none-any.whl", hash = "sha256:07dd1293bac6c77daa5ccdc22c0b41b2315bee0e339a9f035ba86a9f1a272002"},
{file = "humanize-2.4.0.tar.gz", hash = "sha256:42ae7d54b398c01bd100847f6cb0fc9e381c21be8ad3f8e2929135e48dbff026"},
]
idna = [
{file = "idna-2.9-py2.py3-none-any.whl", hash = "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa"},
{file = "idna-2.9.tar.gz", hash = "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb"},
]
multidict = [
{file = "multidict-4.7.6-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:275ca32383bc5d1894b6975bb4ca6a7ff16ab76fa622967625baeebcf8079000"},
{file = "multidict-4.7.6-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:1ece5a3369835c20ed57adadc663400b5525904e53bae59ec854a5d36b39b21a"},
{file = "multidict-4.7.6-cp35-cp35m-win32.whl", hash = "sha256:5141c13374e6b25fe6bf092052ab55c0c03d21bd66c94a0e3ae371d3e4d865a5"},
{file = "multidict-4.7.6-cp35-cp35m-win_amd64.whl", hash = "sha256:9456e90649005ad40558f4cf51dbb842e32807df75146c6d940b6f5abb4a78f3"},
{file = "multidict-4.7.6-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:e0d072ae0f2a179c375f67e3da300b47e1a83293c554450b29c900e50afaae87"},
{file = "multidict-4.7.6-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:3750f2205b800aac4bb03b5ae48025a64e474d2c6cc79547988ba1d4122a09e2"},
{file = "multidict-4.7.6-cp36-cp36m-win32.whl", hash = "sha256:f07acae137b71af3bb548bd8da720956a3bc9f9a0b87733e0899226a2317aeb7"},
{file = "multidict-4.7.6-cp36-cp36m-win_amd64.whl", hash = "sha256:6513728873f4326999429a8b00fc7ceddb2509b01d5fd3f3be7881a257b8d463"},
{file = "multidict-4.7.6-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:feed85993dbdb1dbc29102f50bca65bdc68f2c0c8d352468c25b54874f23c39d"},
{file = "multidict-4.7.6-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:fcfbb44c59af3f8ea984de67ec7c306f618a3ec771c2843804069917a8f2e255"},
{file = "multidict-4.7.6-cp37-cp37m-win32.whl", hash = "sha256:4538273208e7294b2659b1602490f4ed3ab1c8cf9dbdd817e0e9db8e64be2507"},
{file = "multidict-4.7.6-cp37-cp37m-win_amd64.whl", hash = "sha256:d14842362ed4cf63751648e7672f7174c9818459d169231d03c56e84daf90b7c"},
{file = "multidict-4.7.6-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:c026fe9a05130e44157b98fea3ab12969e5b60691a276150db9eda71710cd10b"},
{file = "multidict-4.7.6-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:51a4d210404ac61d32dada00a50ea7ba412e6ea945bbe992e4d7a595276d2ec7"},
{file = "multidict-4.7.6-cp38-cp38-win32.whl", hash = "sha256:5cf311a0f5ef80fe73e4f4c0f0998ec08f954a6ec72b746f3c179e37de1d210d"},
{file = "multidict-4.7.6-cp38-cp38-win_amd64.whl", hash = "sha256:7388d2ef3c55a8ba80da62ecfafa06a1c097c18032a501ffd4cabbc52d7f2b19"},
{file = "multidict-4.7.6.tar.gz", hash = "sha256:fbb77a75e529021e7c4a8d4e823d88ef4d23674a202be4f5addffc72cbb91430"},
]
pillow = [
{file = "Pillow-7.1.2-cp35-cp35m-macosx_10_10_intel.whl", hash = "sha256:ae2b270f9a0b8822b98655cb3a59cdb1bd54a34807c6c56b76dd2e786c3b7db3"},
{file = "Pillow-7.1.2-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:d23e2aa9b969cf9c26edfb4b56307792b8b374202810bd949effd1c6e11ebd6d"},
@ -179,3 +304,22 @@ sqlparse = [
{file = "sqlparse-0.3.1-py2.py3-none-any.whl", hash = "sha256:022fb9c87b524d1f7862b3037e541f68597a730a8843245c349fc93e1643dc4e"},
{file = "sqlparse-0.3.1.tar.gz", hash = "sha256:e162203737712307dfe78860cc56c8da8a852ab2ee33750e33aeadf38d12c548"},
]
yarl = [
{file = "yarl-1.4.2-cp35-cp35m-macosx_10_13_x86_64.whl", hash = "sha256:3ce3d4f7c6b69c4e4f0704b32eca8123b9c58ae91af740481aa57d7857b5e41b"},
{file = "yarl-1.4.2-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:a4844ebb2be14768f7994f2017f70aca39d658a96c786211be5ddbe1c68794c1"},
{file = "yarl-1.4.2-cp35-cp35m-win32.whl", hash = "sha256:d8cdee92bc930d8b09d8bd2043cedd544d9c8bd7436a77678dd602467a993080"},
{file = "yarl-1.4.2-cp35-cp35m-win_amd64.whl", hash = "sha256:c2b509ac3d4b988ae8769901c66345425e361d518aecbe4acbfc2567e416626a"},
{file = "yarl-1.4.2-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:308b98b0c8cd1dfef1a0311dc5e38ae8f9b58349226aa0533f15a16717ad702f"},
{file = "yarl-1.4.2-cp36-cp36m-manylinux1_x86_64.whl", hash = "sha256:944494be42fa630134bf907714d40207e646fd5a94423c90d5b514f7b0713fea"},
{file = "yarl-1.4.2-cp36-cp36m-win32.whl", hash = "sha256:5b10eb0e7f044cf0b035112446b26a3a2946bca9d7d7edb5e54a2ad2f6652abb"},
{file = "yarl-1.4.2-cp36-cp36m-win_amd64.whl", hash = "sha256:a161de7e50224e8e3de6e184707476b5a989037dcb24292b391a3d66ff158e70"},
{file = "yarl-1.4.2-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:26d7c90cb04dee1665282a5d1a998defc1a9e012fdca0f33396f81508f49696d"},
{file = "yarl-1.4.2-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:0c2ab325d33f1b824734b3ef51d4d54a54e0e7a23d13b86974507602334c2cce"},
{file = "yarl-1.4.2-cp37-cp37m-win32.whl", hash = "sha256:e15199cdb423316e15f108f51249e44eb156ae5dba232cb73be555324a1d49c2"},
{file = "yarl-1.4.2-cp37-cp37m-win_amd64.whl", hash = "sha256:2098a4b4b9d75ee352807a95cdf5f10180db903bc5b7270715c6bbe2551f64ce"},
{file = "yarl-1.4.2-cp38-cp38-macosx_10_13_x86_64.whl", hash = "sha256:c9959d49a77b0e07559e579f38b2f3711c2b8716b8410b320bf9713013215a1b"},
{file = "yarl-1.4.2-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:25e66e5e2007c7a39541ca13b559cd8ebc2ad8fe00ea94a2aad28a9b1e44e5ae"},
{file = "yarl-1.4.2-cp38-cp38-win32.whl", hash = "sha256:6faa19d3824c21bcbfdfce5171e193c8b4ddafdf0ac3f129ccf0cdfcb083e462"},
{file = "yarl-1.4.2-cp38-cp38-win_amd64.whl", hash = "sha256:0ca2f395591bbd85ddd50a82eb1fde9c1066fafe888c5c7cc1d810cf03fd3cc6"},
{file = "yarl-1.4.2.tar.gz", hash = "sha256:58cd9c469eced558cd81aa3f484b2924e8897049e06889e8ff2510435b7ef74b"},
]

View file

@ -12,6 +12,7 @@ Pillow = "^7.1.2"
beautifulsoup4 = "^4.9.1"
psutil = "^5.7.0"
humanize = "^2.4.0"
aiohttp = "^3.6.2"
[tool.poetry.dev-dependencies]

View file

@ -1,13 +1,8 @@
import socket
import urllib.request
from enum import Enum
from socket import timeout
from urllib.error import URLError
from urllib.request import Request
from bs4 import BeautifulSoup
from django.db import models
from django.utils.functional import cached_property
from ordered_model.models import OrderedModel
@ -25,44 +20,4 @@ class Service(OrderedModel):
url = models.URLField()
def __str__(self):
return f"{self.short_name} ({self.url})"
@cached_property
def index_request(self):
try:
request = Request(self.url, headers={'User-Agent': 'its just me humble status page'})
return urllib.request.urlopen(request, timeout=1)
except (URLError, timeout):
return None
def get_status(self):
if self.port:
a_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
location = ("localhost", self.port)
result_of_check = a_socket.connect_ex(location)
if result_of_check == 0:
return ServiceStatus.OK
else:
return ServiceStatus.DOWN
if self.index_request and self.index_request.getcode() == 200:
return ServiceStatus.OK
else:
return ServiceStatus.DOWN
@cached_property
def image_or_favicon(self):
if self.image:
return self.image.url
if self.index_request:
parsed_html = BeautifulSoup(self.index_request, features="html.parser")
link_tags = parsed_html.find_all('link')
for rel in ['apple-touch-icon', 'shortcut', 'icon']:
for link_tag in link_tags:
if rel in link_tag.attrs['rel']:
result = link_tag.attrs['href']
if self.url not in result:
return self.url + (result if result.startswith("/") else f"/{result}")
else:
return result
return f"{self.short_name} ({self.url})"

View file

@ -104,6 +104,10 @@ main {
color: darkred;
}
.status-unknown .service-status {
color: darkorange;
}
.stats {
text-align: justify;
padding: 1.5rem 0;

View file

@ -14,10 +14,10 @@
</header>
<main>
{% for service in services %}
<a class="service status-{{ service.get_status.value }}" href="{{ service.url }}">
<a class="service status-{{ service.status }}" href="{{ service.url }}">
<section class="service-content">
{% if service.image_or_favicon %}
<img src="{{ service.image_or_favicon }}" alt="image for {{ service }}"/>
{% if service.image_url %}
<img src="{{ service.image_url }}" alt="image for {{ service.short_name }}"/>
{% endif %}
<div class="label">
<h3>{{ service.short_name }}</h3>
@ -27,7 +27,7 @@
</div>
</section>
<section class="service-status">
STATUS: {{ service.get_status.value }}
STATUS: {{ service.status }}
</section>
</a>
{% endfor %}

View file

@ -1,11 +1,15 @@
import asyncio
import socket
import time
from collections import namedtuple
import aiohttp
import psutil
from bs4 import BeautifulSoup
from django.views.generic import TemplateView
from humanize import naturalsize
from sdbs_infra.dashboard.models import Service
from sdbs_infra.dashboard.models import Service, ServiceStatus
class IndexView(TemplateView):
@ -13,10 +17,63 @@ class IndexView(TemplateView):
def get_context_data(self, **kwargs):
return {
'services': Service.objects.all(),
'services': asyncio.run(self.process_services(list(Service.objects.all()))),
'vps_stats': self.vps_stats()
}
@staticmethod
async def process_services(services):
result = []
session = aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=5, sock_connect=1))
for service in services:
index_status, index_text = None, None
if not service.port or not service.image:
try:
async with session.get(service.url) as response:
index_status, index_text = response.status, await response.text()
except asyncio.TimeoutError:
pass
if service.port:
a_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
location = ("localhost", service.port)
result_of_check = a_socket.connect_ex(location)
if result_of_check == 0:
status = ServiceStatus.OK
else:
status = ServiceStatus.DOWN
elif index_status:
status = ServiceStatus.OK if index_status == 200 else ServiceStatus.DOWN
else:
status = ServiceStatus.UNKNOWN
image = None
if service.image:
image = service.image.url
elif index_text:
parsed_html = BeautifulSoup(index_text, features="html.parser")
link_tags = parsed_html.find_all('link')
for rel in ['apple-touch-icon', 'shortcut', 'icon']:
for link_tag in link_tags:
if rel in link_tag.attrs['rel']:
link = link_tag.attrs['href']
if service.url not in link:
image = service.url + (link if link.startswith("/") else f"/{link}")
else:
image = link
result.append({
'status': status.value,
'image_url': image,
**vars(service)
})
await session.close()
return result
# noinspection PyListCreation
@staticmethod
def vps_stats():