💊 capsul.org cloud compute service - python flask web application
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 

389 lines
16 KiB

import os
import re
import sys
from datetime import datetime, timedelta
import click
from flask.cli import with_appcontext
from flask import Blueprint
from flask import current_app
from psycopg2 import ProgrammingError
from flask_mail import Message
from capsulflask.db import get_model
from capsulflask.shared import my_exec_info_message, get_account_balance
from capsulflask.payment import check_if_shortterm_flag_can_be_unset
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information
bp = Blueprint('cli', __name__)
@bp.cli.command('sql')
@click.option('-f', help='script filename')
@click.option('-c', help='sql command')
@with_appcontext
def sql_script(f, c):
"""Run a sql script against the database. script is run 1 command at a time inside a single transaction."""
model = get_model()
script = ""
if f:
filepath = os.path.join(os.getcwd(), f)
if not os.path.isfile(filepath):
raise f"{filepath} is not a file"
with open(filepath, 'rb') as file:
script = file.read().decode("utf8")
elif c:
script = c
else:
click.echo(f"you must provide sql to run either inline with the -c argument or in a file with the -f argument")
return
commands = re.split(";\\s+", script)
for command in commands:
if command.strip() != "":
model.cursor.execute(command)
if re.match("^\\s*select", command, re.IGNORECASE) is not None:
for row in model.cursor.fetchall():
def format_value(x):
if isinstance(x, bool):
return "true" if x else "false"
if not x :
return "null"
if isinstance(x, datetime):
return x.isoformat()
return f"{x}"
click.echo(", ".join(list(map(format_value, row))))
else:
click.echo(f"{model.cursor.rowcount} rows affected.")
model.connection.commit()
@bp.cli.command('account-balance')
@click.option('-u', help='users email address')
@with_appcontext
def account_balance(u):
vms = get_model().list_vms_for_account(u)
payments = get_model().list_payments_for_account(u)
click.echo(".")
click.echo(".")
click.echo(get_account_balance(vms, payments, datetime.utcnow()))
click.echo(".")
@bp.cli.command('cron-task')
@with_appcontext
def cron_task():
# make sure btcpay payments get completed (in case we miss a webhook), otherwise invalidate the payment
current_app.logger.info("cron_task: starting clean_up_unresolved_btcpay_invoices")
clean_up_unresolved_btcpay_invoices()
current_app.logger.info("cron_task: finished clean_up_unresolved_btcpay_invoices")
# notify when funds are about to run out and delete long-term vms once account reaches -$10
current_app.logger.info("cron_task: starting notify_users_about_account_balance")
notify_users_about_account_balance()
current_app.logger.info("cron_task: finished notify_users_about_account_balance")
# delete short-term vms and notify user once account reaches $0
current_app.logger.info("cron_task: starting delete_shortterm_vms_if_account_is_empty")
delete_shortterm_vms_if_account_is_empty()
current_app.logger.info("cron_task: finished delete_shortterm_vms_if_account_is_empty")
# make sure vm system and DB are synced
current_app.logger.info("cron_task: starting ensure_vms_and_db_are_synced")
ensure_vms_and_db_are_synced()
current_app.logger.info("cron_task: finished ensure_vms_and_db_are_synced")
def clean_up_unresolved_btcpay_invoices():
unresolved_btcpay_invoices = get_model().get_unresolved_btcpay_invoices()
for unresolved_invoice in unresolved_btcpay_invoices:
invoice_id = unresolved_invoice['id']
btcpay_invoice = None
try:
btcpay_invoice = current_app.config['BTCPAY_CLIENT'].get_invoice(invoice_id)
except:
current_app.logger.error(f"""
error was thrown when contacting btcpay server for invoice {invoice_id}:
{my_exec_info_message(sys.exc_info())}"""
)
continue
days = float((datetime.now() - unresolved_invoice['created']).total_seconds())/float(60*60*24)
if btcpay_invoice['status'] == "complete":
current_app.logger.info(
f"resolving btcpay invoice {invoice_id} "
f"({unresolved_invoice['email']}, ${unresolved_invoice['dollars']}) as completed "
)
resolved_invoice_email = get_model().btcpay_invoice_resolved(invoice_id, True)
if resolved_invoice_email is not None:
check_if_shortterm_flag_can_be_unset(resolved_invoice_email)
elif days >= 1:
current_app.logger.info(
f"resolving btcpay invoice {invoice_id} "
f"({unresolved_invoice['email']}, ${unresolved_invoice['dollars']}) as invalidated, "
f"btcpay server invoice status: {btcpay_invoice['status']}"
)
get_model().btcpay_invoice_resolved(invoice_id, False)
get_model().delete_payment_session("btcpay", invoice_id)
delete_at_account_balance_dollars = -10
def get_warning_headline(warning_id, pluralize_capsul):
return dict(
zero_1w= (
"According to our calculations, your Capsul account will run out of funds before this time next week.\n\n"
),
zero_1d= (
"According to our calculations, your Capsul account will run out of funds by this time tomorrow.\n\n"
),
zero_now= (
f"You have run out of funds! You will no longer be able to create Capsuls.\n\n"
f"As a courtesy, we'll let your existing Capsul{pluralize_capsul} keep running until your account "
"reaches a -$10 balance, at which point they will be deleted.\n\n"
),
delete_1w= (
"You have run out of funds and have not refilled your account.\n\n"
f"As a courtesy, we've let your existing Capsul{pluralize_capsul} keep running. "
f"However, your account will reach a -$10 balance some time next week and your Capsul{pluralize_capsul} "
"will be deleted.\n\n"
),
delete_1d= (
"You have run out of funds and have not refilled your account.\n\n"
f"As a courtesy, we have let your existing Capsul{pluralize_capsul} keep running. "
f"However, your account will reach a -$10 balance by this time tomorrow and "
f"your Capsul{pluralize_capsul} will be deleted.\n\n"
f"Last chance to deposit funds now and keep your Capsul{pluralize_capsul} running! "
),
delete_now= (
f"Your account reached a -$10 balance and your Capsul{pluralize_capsul} were deleted."
)
)[warning_id]
def get_warnings_list():
return [
dict(
id='zero_1w',
get_active=lambda balance_1w, balance_1d, balance_now: balance_1w < 0,
get_subject=lambda _: "Capsul One Week Payment Reminder",
get_body=lambda base_url, pluralize_capsul: (
f"{get_warning_headline('zero_1w', pluralize_capsul)}"
f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n"
"If you believe you have recieved this message in error, please let us know: support@cyberia.club"
)
),
dict(
id='zero_1d',
get_active=lambda balance_1w, balance_1d, balance_now: balance_1d < 0,
get_subject=lambda _: "Capsul One Day Payment Reminder",
get_body=lambda base_url, pluralize_capsul: (
f"{get_warning_headline('zero_1d', pluralize_capsul)}"
f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n"
"If you believe you have recieved this message in error, please let us know: support@cyberia.club"
)
),
dict(
id='zero_now',
get_active=lambda balance_1w, balance_1d, balance_now: balance_now < 0,
get_subject=lambda _: "Your Capsul Account is No Longer Funded",
get_body=lambda base_url, pluralize_capsul: (
f"{get_warning_headline('zero_now', pluralize_capsul)}"
f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n"
f"If you need help decomissioning your Capsul{pluralize_capsul}, "
"would like to request backups, or de-activate your account, please contact: support@cyberia.club"
)
),
dict(
id='delete_1w',
get_active=lambda balance_1w, balance_1d, balance_now: balance_1w < delete_at_account_balance_dollars,
get_subject=lambda pluralize_capsul: f"Your Capsul{pluralize_capsul} Will be Deleted In Less Than a Week",
get_body=lambda base_url, pluralize_capsul: (
f"{get_warning_headline('delete_1w', pluralize_capsul)}"
f"Log in now to re-fill your account! {base_url}/console/account-balance\n\n"
f"If you need help decomissioning your Capsul{pluralize_capsul}, "
"would like to request backups, or de-activate your account, please contact: support@cyberia.club"
)
),
dict(
id='delete_1d',
get_active=lambda balance_1w, balance_1d, balance_now: balance_1d < delete_at_account_balance_dollars,
get_subject=lambda pluralize_capsul: f"Last Chance to Save your Capsul{pluralize_capsul}: Gone Tomorrow",
get_body=lambda base_url, pluralize_capsul: (
f"{get_warning_headline('delete_1d', pluralize_capsul)}"
f"{base_url}/console/account-balance"
)
),
dict(
id='delete_now',
get_active=lambda balance_1w, balance_1d, balance_now: balance_now < delete_at_account_balance_dollars,
get_subject=lambda pluralize_capsul: f"Capsul{pluralize_capsul} Deleted",
get_body=lambda base_url, pluralize_capsul: (
f"{get_warning_headline('delete_now', pluralize_capsul)}"
)
)
]
def notify_users_about_account_balance():
accounts = get_model().all_accounts()
out_of_bounds_accounts = dict()
for account in accounts:
vms = get_model().list_vms_for_account(account['email'])
payments = get_model().list_payments_for_account(account['email'])
balance_1w = get_account_balance(vms, payments, datetime.utcnow() + timedelta(days=7))
balance_1d = get_account_balance(vms, payments, datetime.utcnow() + timedelta(days=1))
balance_now = get_account_balance(vms, payments, datetime.utcnow())
current_warning = account['account_balance_warning']
if balance_now < -11 and len(list(filter(lambda vm: not vm['deleted'], vms))) > 0:
out_of_bounds_accounts[account['email']] = balance_now
longterm_vms = list(filter(lambda vm: vm['shortterm'] == False, vms))
if len(longterm_vms) == 0:
continue
pluralize_capsul = "s" if len(longterm_vms) > 1 else ""
warnings = get_warnings_list()
current_warning_index = -1
if current_warning:
for i in range(0, len(warnings)):
if warnings[i]['id'] == current_warning:
current_warning_index = i
index_to_send = -1
for i in range(0, len(warnings)):
if i > current_warning_index and warnings[i]['get_active'](balance_1w, balance_1d, balance_now):
index_to_send = i
if index_to_send > -1:
current_app.logger.info(f"cron_task: sending {warnings[index_to_send]['id']} warning email to {account['email']}.")
get_body = warnings[index_to_send]['get_body']
get_subject = warnings[index_to_send]['get_subject']
current_app.config["FLASK_MAIL_INSTANCE"].send(
Message(
get_subject(pluralize_capsul),
body=get_body(current_app.config['BASE_URL'], pluralize_capsul),
sender=current_app.config["MAIL_DEFAULT_SENDER"],
recipients=[account['email']]
)
)
get_model().set_account_balance_warning(account['email'], warnings[index_to_send]['id'])
if index_to_send == len(warnings)-1:
for vm in longterm_vms:
current_app.logger.warning(f"cron_task: deleting {vm['id']} ( {account['email']} ) due to negative account balance.")
current_app.config["HUB_MODEL"].destroy(email=account["email"], id=vm['id'])
get_model().delete_vm(email=account["email"], id=vm['id'])
if len(out_of_bounds_accounts) > 0:
lines_redacted = ["The following accounts have out-of-bounds account balances: (Un-redacted email addresses avaliable in the logs)", ""]
lines = ["The following accounts have out-of-bounds account balances:", ""]
for email, balance in out_of_bounds_accounts.items():
lines.append(f"{email}: ${format(balance, '.2f')}")
lines_redacted.append(f"*******: ${format(balance, '.2f')}")
email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
current_app.logger.info(f"notify_users_about_account_balance: sending out of bounds account balances email to {','.join(email_addresses)}:")
for line in lines:
current_app.logger.info(f"notify_users_about_account_balance: {line}.")
current_app.config["FLASK_MAIL_INSTANCE"].send(
Message(
"Capsul Out Of Bounds Account Balance Notification",
sender=current_app.config["MAIL_DEFAULT_SENDER"],
body="\n".join(lines_redacted),
recipients=email_addresses
)
)
def delete_shortterm_vms_if_account_is_empty():
accounts = get_model().all_accounts()
for account in accounts:
vms = get_model().list_vms_for_account(account['email'])
payments = get_model().list_payments_for_account(account['email'])
balance = get_account_balance(vms, payments, datetime.utcnow())
shortterm_vms = list(filter(lambda vm: vm['shortterm'] == True and not vm['deleted'], vms))
if len(shortterm_vms) > 0 and balance <= 0:
pluralize_capsul = "s" if len(shortterm_vms) > 1 else ""
pluralize_past_tense = "have" if len(shortterm_vms) > 1 else "has"
current_app.config["FLASK_MAIL_INSTANCE"].send(
Message(
f"Short-term Capsul{pluralize_capsul} Deleted",
body=(
f"You have run out of funds! Your Short-term Capsul{pluralize_capsul} {pluralize_past_tense} been deleted.\n\n"
),
sender=current_app.config["MAIL_DEFAULT_SENDER"],
recipients=[account['email']]
)
)
for vm in shortterm_vms:
current_app.logger.warning(f"cron_task: deleting shortterm vm {vm['id']} ( {account['email']} ) due to negative account balance.")
current_app.config["HUB_MODEL"].destroy(email=account["email"], id=vm['id'])
get_model().delete_vm(email=account["email"], id=vm['id'])
def ensure_vms_and_db_are_synced():
db_vms_by_id = get_all_vms_from_db()
virt_vms_by_id = get_all_vms_from_hosts(db_vms_by_id)
inconsistency_info = get_inconsistent_capsuls_information(db_vms_by_id, virt_vms_by_id)
errors = list()
for vm in inconsistency_info['in_db_but_not_in_virt']:
errors.append(f"{vm['id']} ({vm['email']}) is in the database but not in the virtualization model")
for vm in inconsistency_info['state_not_equal_to_desired_state']:
errors.append(f"{vm['id']} ({vm['email']}) is {vm['state']} but it is supposed to be {vm['desired_state']}")
for vm in inconsistency_info['stole_someone_elses_ip_and_own_ip_avaliable']:
errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_and_own_ip_avaliable current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
for vm in inconsistency_info['stole_someone_elses_ip_but_own_ip_also_stolen']:
errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_but_own_ip_also_stolen current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
for vm in inconsistency_info['has_wrong_ip']:
errors.append(f"{vm['id']} ({vm['email']}) has_wrong_ip current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
if len(errors) > 0:
email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
for error in errors:
current_app.logger.info(f"cron_task: {error}.")
current_app.config["FLASK_MAIL_INSTANCE"].send(
Message(
"Capsul Consistency Check Failed",
sender=current_app.config["MAIL_DEFAULT_SENDER"],
body="\n".join(errors),
recipients=email_addresses
)
)