Commit cb0532e2 authored by Davide Depau's avatar Davide Depau

Merge branch 'presence-graph' into 'master'

Presence graph

See merge request b4/bits-server!3
parents f2613fd6 0b08e5b5
......@@ -93,19 +93,6 @@ will be able to log data and change status.
TODOs
=====
Presence forecast
-----------------
Fede.tft and Otacon22 BITS developers came up with a great idea:
since the university timetables are fixed in a span of six months and most
students will come to POuL during breaks between lessons, we can forecast
presence by processing data from past few weeks.
The forecast is plot as a green to red table, each shade corresponding to
a decreasing probability of finding someone at the given time and day.
![Presence graph](/bits_presence.png)
MAC address detection
---------------------
......
......@@ -10,36 +10,34 @@
Serve content to clients via TCP. All HTTP, WS etc. handlers belong on here.
"""
import tornado.web
import tornado.httpserver
import tornado.web
from tornado.options import options
from bitsd.common import LOG, bind
from . import auth
from . import handlers
from . import uimodules
from . import auth
from bitsd.common import LOG, bind
def start():
"""Setup HTTP/WS server. **MUST** be called prior to any operation."""
application = tornado.web.Application([
# FIXME daltonism workaround, should be implemented client-side
(r'/(?:|blind)', handlers.HomePageHandler),
(r'/log', handlers.LogPageHandler),
(r'/status', handlers.StatusPageHandler),
(r'/data', handlers.DataPageHandler),
(r'/presence', handlers.PresenceForecastHandler),
(r'/(info)', handlers.MarkdownPageHandler),
(r'/ws', handlers.StatusHandler),
(r'/login', handlers.LoginPageHandler),
(r'/logout', handlers.LogoutPageHandler),
(r'/admin', handlers.AdminPageHandler),
(r'/message', handlers.MessagePageHandler),
(r'/data.php', handlers.RTCHandler),
(r'/macupdate', handlers.MACUpdateHandler),
],
# FIXME daltonism workaround, should be implemented client-side
(r'/(?:|blind)', handlers.HomePageHandler),
(r'/log', handlers.LogPageHandler),
(r'/status', handlers.StatusPageHandler),
(r'/data', handlers.DataPageHandler),
(r'/(info)', handlers.MarkdownPageHandler),
(r'/ws', handlers.StatusHandler),
(r'/login', handlers.LoginPageHandler),
(r'/logout', handlers.LogoutPageHandler),
(r'/admin', handlers.AdminPageHandler),
(r'/message', handlers.MessagePageHandler),
(r'/data.php', handlers.RTCHandler),
(r'/macupdate', handlers.MACUpdateHandler),
(r'/presence.svg', handlers.PresenceGraphHandler)
],
ui_modules=uimodules,
gzip=True,
debug=options.developer_mode,
......
......@@ -259,3 +259,11 @@ ul#log-status {
.nav-open nav.navbar.fixed-top {
z-index: 2001;
}
.card .card-body.card-no-padding {
padding: 0;
}
img#presence_graph {
border-radius: 6px;
}
......@@ -10,11 +10,11 @@
"""
HTTP requests handlers.
"""
import json
import time
from datetime import datetime, timedelta
import json
import markdown
import time
import tornado.auth
import tornado.websocket
from sqlalchemy import distinct
......@@ -28,9 +28,9 @@ import bitsd.persistence.query as query
from bitsd.common import LOG, secure_compare
from bitsd.persistence.engine import session_scope, persist
from bitsd.persistence.models import Status, User, MACToUser, LoginAttempt
from bitsd.server.presence_graph.presence import Presence
from .auth import verify, DoSError
from .notifier import MessageNotifier
from .presence import PresenceForecaster
def cache(seconds):
......@@ -342,19 +342,6 @@ class AdminPageHandler(BaseHandler):
)
class PresenceForecastHandler(BaseHandler):
"""Handler for presence stats.
Upon GET, it will render JSON-encoded probabilities,
as a 2D array (forecast for each weekday, at 30min granularity)."""
FORECASTER = PresenceForecaster()
@cache(86400)
def get(self):
data = self.FORECASTER.forecast()
self.write({"forecast": data})
self.finish()
class MessagePageHandler(BaseHandler):
@tornado.web.authenticated
def get(self):
......@@ -437,3 +424,15 @@ class MACUpdateHandler(BaseHandler):
def check_xsrf_cookie(self):
# Since this is an API call, we need to disable anti-XSRF protection
pass
class PresenceGraphHandler(BaseHandler):
"""Handler for the presence graph"""
PRESENCE = Presence()
def prepare(self) -> None:
self.set_header("Content-Type", "image/svg+xml")
def get(self) -> None:
self.write(PresenceGraphHandler.PRESENCE.get_graph().tostring())
#
# Copyright (C) 2013 Stefano Sanfilippo
# Copyright (C) 2013 BITS development team
#
# This file is part of bitsd, which is released under the terms of
# GNU GPLv3. See COPYING at top level for more information.
#
from bitsd.persistence.engine import session_scope
from bitsd.persistence.models import Status
from sqlalchemy import asc
class PresenceForecaster:
class InvalidResolutionError(Exception):
def __init__(self):
self.message = "Resolution must be a submultiple of 60 minutes!"
def __init__(self, resolution=30, samples_cont=5000):
if self.resolution_is_invalid(resolution):
raise self.InvalidResolutionError()
self.samples_count = samples_cont
self.ticks_per_hour = 60 / resolution
self.minutes_per_tick = resolution
def forecast(self):
#TODO caching
return self.calculate_frequencies()
def calculate_frequencies(self):
samples = self.get_samples()
buckets = self.count_presence_per_slot(samples)
return self.normalize(buckets)
def count_presence_per_slot(self, samples):
buckets = self.init_buckets()
# TODO algorithm here
return buckets
def init_buckets(self):
return [[0] * (24 * self.ticks_per_hour) for i in range(7)]
def get_samples(self):
with session_scope() as session:
samples = session \
.query(Status.timestamp, Status.value) \
.filter((Status.value == Status.OPEN) | (Status.value == Status.CLOSED)) \
.order_by(asc(Status.timestamp)) \
.limit(self.samples_count)
offset = self.first_open_offset(samples)
return samples[offset:]
def first_open_offset(self, samples):
offset = 0
while samples[offset].value != Status.OPEN:
offset += 1
return offset
@staticmethod
def resolution_is_invalid(resolution):
return (60 % resolution) != 0
def calculate_coordinates(self, sample):
timestamp = sample.timestamp
weekday = timestamp.weekday()
timeslot = (self.ticks_per_hour * timestamp.hour) + int(1. * timestamp.minute / self.minutes_per_tick)
return weekday, timeslot
def normalize(self, buckets):
for day in buckets:
for i, slot in enumerate(day):
day[i] = 1. * slot / self.samples_count
return buckets
\ No newline at end of file
from .parameters import *
from .graph import *
from .presence import *
\ No newline at end of file
from datetime import time
from typing import List
from colour import Color
from svgwrite import Drawing
from svgwrite.container import Group, Style
from svgwrite.gradients import LinearGradient
from svgwrite.shapes import Line, Rect
from svgwrite.text import Text
from bitsd.server.presence_graph.parameters import START_TIME, END_TIME, COLOR_CLOSE, COLOR_OPEN
HEIGHT = 1000
WIDTH = 1000
HEADER_SIZE = 70
GRADIENT_NUMBER_COLORS = 11
class PresenceGraph(Drawing):
def __init__(self, data: List[List[float]], data_header: List[float] = None, *args, **kwargs):
super().__init__(profile='full', debug=True, *args, **kwargs)
# Set the viewbox instead of the size
self.viewbox(width=WIDTH, height=HEIGHT)
# Save the data
self._data = data
self._data_header = data_header
# Calculate the discrete gradient
self._gradient = list(COLOR_CLOSE.range_to(COLOR_OPEN, GRADIENT_NUMBER_COLORS))
def generate(self) -> None:
"""Generates the graph with the provided data"""
self._mk_gradients()
self._mk_header_style()
self._add_header()
self._add_rect()
self._add_hours()
self._add_grid()
def _mk_header_style(self) -> None:
# Set default background for the header
self.defs.add(Style("#header {fill: black;}"))
if self._data_header is not None or True:
for i in range(5):
value = int(round(self._data_header[i] * (GRADIENT_NUMBER_COLORS - 1)))
self.defs.add(Style("#day_{} {{fill: {};}}".format(i, self._gradient[value].hex)))
def _mk_gradients(self) -> None:
"""Creates the gradient for the graph"""
# For each day of the week
for i in range(0, 5):
# Make a vertical linear gradient
lg = LinearGradient((0, 0), (0, 1), id="grad_{}".format(i))
# Add the gradient to the svg
self.defs.add(lg)
# For each data for the single day
for j in range(len(self._data[i])):
# Calculate the position in the gradient
position = j / len(self._data[i])
# Calculate the rescaled value on the number of the colors in the gradient
value = int(round(self._data[i][j] * (GRADIENT_NUMBER_COLORS - 1)))
# Get the hex value of the computed color
color = self._gradient[value].hex
# Add the color in the right position of the gradient
lg.add_stop_color(position, color)
def _add_rect(self) -> None:
"""Adds the colored block to the graph"""
# Create a group for the blocks of the graph
g = self.add(Group())
# Add the blocks of the graph and link them to the gradients
for i in range(0, 5):
g.add(Rect((i * WIDTH / 5, HEADER_SIZE), (WIDTH / 5, HEIGHT), fill="url(#grad_{})".format(i)))
def _add_grid(self) -> None:
"""Adds the grid to the svg"""
# Create a group for the grid
g = self.add(Group(id="grid",
stroke="black",
stroke_width="5"
))
# Add the vertical lines for the grid
for i in range(1, 5):
g.add(Line((i * WIDTH / 5, 0), (i * WIDTH / 5, HEIGHT)))
# Add the line of the header
g.add(Line((0, HEADER_SIZE), (WIDTH, HEADER_SIZE)))
def _add_hours(self) -> None:
"""Adds the hours labels and the lines to the svg"""
# Create a group for the lines of the hours
gl = self.add(Group(id="hours_lines",
stroke="black",
stroke_width="2"
))
# Create a group for the labels of the hours
g = self.add(Group(id="hours",
text_anchor="middle",
font_size="25",
font_family="'Open Sans', Sans",
font_weight="bold"
))
number_hours = END_TIME.hour - START_TIME.hour
# Calculate the vertical position for the labels
for i, y in enumerate([j * (HEIGHT - HEADER_SIZE) / number_hours for j in range(1, number_hours)]):
# Create the lines for the hours
gl.add(Line((0, HEADER_SIZE + y),
(WIDTH / 2 - 50, HEADER_SIZE + y)))
gl.add(Line((WIDTH / 2 + 50, HEADER_SIZE + y),
(WIDTH, HEADER_SIZE + y)))
# Create the labels of the hours
g.add(Text(time(hour=(i + START_TIME.hour + 1)).strftime("%H:%M"),
x=[WIDTH / 2], y=[HEADER_SIZE + y], dy=[9]))
def _add_header(self) -> None:
"""Adds the colored header blocks and their labels"""
# Create a group for the colored header blocks
g = self.add(Group(id="header"))
# Create the header blocks
for i in range(5):
g.add(Rect((i * WIDTH / 5, 0), (WIDTH / 5, HEADER_SIZE), id="day_{}".format(i)))
# Create a group for the labels of the header
g = self.add(Group(id="days",
text_anchor="middle",
font_size="30",
font_family="'Open Sans', Sans",
font_weight="bold"
))
# Create the days labels
for i, day in enumerate(["Mon", "Tue", "Wed", "Thu", "Fri"]):
g.add(Text(day, x=[(i * 2 + 1) * WIDTH / 10], y=[HEADER_SIZE / 2], dy=[13]))
from datetime import timedelta, time
# Time of the start of the day
from colour import Color
START_TIME = time(hour=8)
# Time of the end of the day
END_TIME = time(hour=21)
# Delta time between two sample
DELTA_SAMPLE = timedelta(minutes=15)
# Size of the window for the average
WINDOW_WIDTH = timedelta(weeks=8)
# Weights of the data of the weeks from the closer to the farther
# They must be at least as many as the number of the weeks in the window
DATA_WEIGHTS = [1000, 300, 200, 150, 100, 70, 30, 20]
# The valid in time for the cache of the graph
GRAPH_CACHE_DURATION = timedelta(hours=6)
# The color for the graph
COLOR_OPEN = Color("#387002")
COLOR_CLOSE = Color("#d32f2f")
\ No newline at end of file
from datetime import datetime, timedelta, time, date
from typing import Tuple, List, Union
import numpy
from sqlalchemy import asc
from bitsd.persistence.engine import session_scope
from bitsd.persistence.models import Status
from .graph import PresenceGraph
from .parameters import WINDOW_WIDTH, START_TIME, END_TIME, DELTA_SAMPLE, DATA_WEIGHTS, GRAPH_CACHE_DURATION
class Presence():
def __init__(self):
# On start the cache is not valid
self._cache_expiration = datetime(year=1970, month=1, day=1)
self._cache = None
def _get_raw_data_from_db(self) -> List[List[List[Tuple[time, bool]]]]:
"""Return a list of day, each day is a list of week, each week is a list of tuples of timestamp and status"""
# Calculate the window for the data
window_date = (
datetime.combine(datetime.now().date(), time()) - WINDOW_WIDTH,
datetime.combine(datetime.now().date(), time()))
# Prepare the data struct
data = [[], [], [], [], []]
# Open a db session
with session_scope() as session:
# For each day of the week
for i in range(7):
# Backwards from today parse the days
current_day = window_date[1] - timedelta(days=(1 + i))
# Skip saturday and sunday (the data are useless)
if (current_day.weekday() == 5 or current_day.weekday() == 6): continue
# For each day of the week in the window
while current_day > window_date[0]:
# Calculate the limit of the day
current_limit = (
datetime.combine(current_day, START_TIME),
datetime.combine(current_day, END_TIME),
)
day = []
# Extract the data from the db
for row in session.query(Status) \
.filter(Status.timestamp.between(*current_limit)).order_by(asc(Status.timestamp)).all():
day.append((row.timestamp.time(), True if row.value == row.OPEN else False))
# Add the data for the current week and the current day of the week
data[current_day.weekday()].append(day)
# Go to the previous week
current_day = current_day - timedelta(weeks=1)
return data
def get_data(self) -> List[List[List[int]]]:
"""Provides the data on presence distribute over weeks"""
# Prepare the structure for the data
data = [[], [], [], [], []]
# Get the raw data from the db
raw_data = self._get_raw_data_from_db()
# For each day of the week
for i in range(5):
# Get the data for the day
day_data = raw_data[i]
# Set the current time to the start of the day
current_time = datetime.combine(date.today(), START_TIME)
# Until the current time is earlier than the end of the day
while current_time.time() <= END_TIME:
# Create a list of weeks' values for a single sample in a single day
status_hour = []
# For each week value
for week in day_data:
# Add to the status_hour list the status for the current time for the current week data
status_hour.append(1 if self._status(week, current_time.time()) else 0)
data[i].append(status_hour)
# Forward to the next time for a sample
current_time = current_time + DELTA_SAMPLE
return data
def get_prevision(self, with_average=False) -> Union[List[List[float]], Tuple[List[List[float]], List[float]]]:
"""Provides prevision data on the HQ's status distribute over hours"""
# Get the data for the weeks
raw_data = self.get_data()
# Prepare the data struct
data = [[], [], [], [], []]
# For each day of the week
for i in range(5):
# Get the data of the current day
day_data = raw_data[i]
# Calculate the weighted average over the data weeks
for j, _ in enumerate(day_data):
data[i].append(numpy.average(day_data[j], weights=DATA_WEIGHTS[:len(day_data[j])]))
# If it's required a average for the days it's provided
if with_average:
average = []
# Calculate the average for the days
for i in range(5):
average.append(numpy.average(data[i]))
return data, average
else:
return data
def _status(self, data: List[Tuple[time, bool]], hour: time) -> bool:
"""Provided data raw data for a single day and a moment returns True if the HQ was opening in that moment"""
status = False
# Check the status until the data are ended or a status following the moment required
for i in [(time(hour=0), False), *data]:
if i[0] > hour:
return status
status = i[1]
return status
def get_graph(self) -> PresenceGraph:
"""Returns the graph of the presence"""
# Check if the cache is expired
if self._cache_expiration < datetime.now():
# Generate a new graph and save it in the cache
self._cache = PresenceGraph(*self.get_prevision(True))
self._cache.generate()
# Update the cache expiration time
self._cache_expiration = datetime.now() + GRAPH_CACHE_DURATION
return self._cache
......@@ -56,7 +56,7 @@
<div class="col col-12 col-lg-6">
<div class="card bg-black">
<div class="card-body">
<div class="card-body card-no-padding">
{% module PresenceWidget() %}
</div>
</div>
......@@ -74,4 +74,4 @@
</div>
#}
{% end %}
\ No newline at end of file
{% end %}
......@@ -73,7 +73,7 @@ class PresenceWidget(tornado.web.UIModule):
def render(self):
#TODO samples = get_latest_statuses(5000)
#TODO + TODO gray
return '<img src="bits_presence.png" alt="Grafico delle presenze" id="presence_graph"/>'
return '<img src="/presence.svg" alt="Grafico delle presenze" id="presence_graph"/>'
class PaginatorWidget(tornado.web.UIModule):
......
......@@ -3,5 +3,8 @@ markdown
sqlalchemy>=0.7
tornado==4.*
paho-mqtt
mysqlclient
#mysqlclient
psycopg2
svgwrite~=1.3
colour~=0.1
numpy~=1.17
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment