Browse Source

crontask: monitor db vs virt model of all capsuls like admin panel

console-access
forest 7 months ago
parent
commit
b013f9c975
  1. 61
      capsulflask/admin.py
  2. 78
      capsulflask/cli.py
  3. 103
      capsulflask/consistency.py

61
capsulflask/admin.py

@ -12,7 +12,7 @@ from nanoid import generate
from capsulflask.metrics import durations as metric_durations
from capsulflask.auth import admin_account_required
from capsulflask.db import get_model
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information
from capsulflask.shared import my_exec_info_message
bp = Blueprint("admin", __name__, url_prefix="/admin")
@ -181,50 +181,7 @@ def index():
display_hosts.append(display_host)
# Now creating the capsul consistency / running status ui
#
virt_vm_id_by_ipv4 = dict()
for vm_id, virt_vm in virt_vms_by_id.items():
if 'public_ipv4' in virt_vm and virt_vm['public_ipv4'] != "":
virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id
db_vm_id_by_ipv4 = dict()
for vm_id, db_vm in db_vms_by_id.items():
if 'public_ipv4' in db_vm and db_vm['public_ipv4'] != "":
db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id
in_db_but_not_in_virt = []
state_not_equal_to_desired_state = []
has_no_desired_ip_address = []
has_not_aquired_ip_address_yet = []
stole_someone_elses_ip_and_own_ip_avaliable = []
stole_someone_elses_ip_but_own_ip_also_stolen = []
has_wrong_ip = []
for vm_id, db_vm in db_vms_by_id.items():
if vm_id not in virt_vms_by_id:
in_db_but_not_in_virt.append(db_vm)
elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]:
db_vm["state"] = virt_vms_by_id[vm_id]['state']
state_not_equal_to_desired_state.append(db_vm)
elif 'public_ipv4' not in db_vm or db_vm["public_ipv4"] == "":
has_no_desired_ip_address.append(db_vm)
elif db_vm["desired_state"] == "running" and ('public_ipv4' not in virt_vms_by_id[vm_id] or virt_vms_by_id[vm_id]['public_ipv4'] == ""):
has_not_aquired_ip_address_yet.append(db_vm)
elif db_vm["desired_state"] == "running" and virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]:
db_vm["desired_ipv4"] = db_vm["public_ipv4"]
db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4']
if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4:
if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4:
stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm)
else:
stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm)
has_wrong_ip.append(db_vm)
# current_app.logger.info(f"list_of_networks: {json.dumps(list_of_networks)}")
inconsistency_info = get_inconsistent_capsuls_information(db_vms_by_id, virt_vms_by_id)
csp_inline_style_nonce = generate(alphabet="1234567890qwertyuiopasdfghjklzxcvbnm", size=10)
@ -235,13 +192,13 @@ def index():
network_display_width_px=network_display_width_px,
csp_inline_style_nonce=csp_inline_style_nonce,
inline_style='\n'.join(inline_styles),
in_db_but_not_in_virt=in_db_but_not_in_virt,
state_not_equal_to_desired_state=state_not_equal_to_desired_state,
has_no_desired_ip_address=has_no_desired_ip_address,
has_not_aquired_ip_address_yet=has_not_aquired_ip_address_yet,
stole_someone_elses_ip_and_own_ip_avaliable=stole_someone_elses_ip_and_own_ip_avaliable,
stole_someone_elses_ip_but_own_ip_also_stolen=stole_someone_elses_ip_but_own_ip_also_stolen,
has_wrong_ip=has_wrong_ip
in_db_but_not_in_virt=inconsistency_info['in_db_but_not_in_virt'],
state_not_equal_to_desired_state=inconsistency_info['state_not_equal_to_desired_state'],
has_no_desired_ip_address=inconsistency_info['has_no_desired_ip_address'],
has_not_aquired_ip_address_yet=inconsistency_info['has_not_aquired_ip_address_yet'],
stole_someone_elses_ip_and_own_ip_avaliable=inconsistency_info['stole_someone_elses_ip_and_own_ip_avaliable'],
stole_someone_elses_ip_but_own_ip_also_stolen=inconsistency_info['stole_someone_elses_ip_but_own_ip_also_stolen'],
has_wrong_ip=inconsistency_info['has_wrong_ip']
)
response = make_response(response_text)

78
capsulflask/cli.py

@ -14,6 +14,7 @@ from flask_mail import Message
from capsulflask.db import get_model
from capsulflask.shared import my_exec_info_message
from capsulflask.console import get_account_balance
from capsulflask.consistency import get_all_vms_from_db, get_all_vms_from_hosts, get_inconsistent_capsuls_information
bp = Blueprint('cli', __name__)
@ -268,46 +269,41 @@ def notify_users_about_account_balance():
def ensure_vms_and_db_are_synced():
pass
# db_vms = get_model().all_vm_ids_with_desired_state()
# # TODO replaced
# #virt_vms = current_app.config["HUB_MODEL"].virsh_list()
# db_ids_dict = dict()
# virt_ids_dict = dict()
# for vm in db_vms:
# db_ids_dict[vm['id']] = vm['desired_state']
# for vm in virt_vms:
# virt_ids_dict[vm['id']] = vm['desired_state']
# errors = list()
# for id in db_ids_dict:
# if id not in virt_ids_dict:
# errors.append(f"{id} is in the database but not in the virtualization model")
# elif db_ids_dict[id] != virt_ids_dict[id]:
# errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
# for id in virt_ids_dict:
# if id not in db_ids_dict:
# errors.append(f"{id} is in the virtualization model but not in the database")
# if len(errors) > 0:
# email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
# email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
# current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
# for error in errors:
# current_app.logger.info(f"cron_task: {error}.")
db_vms_by_id = get_all_vms_from_db()
virt_vms_by_id = get_all_vms_from_hosts(db_vms_by_id)
inconsistency_info = get_inconsistent_capsuls_information(db_vms_by_id, virt_vms_by_id)
errors = list()
for vm in inconsistency_info['in_db_but_not_in_virt']:
errors.append(f"{vm['id']} ({vm['email']}) is in the database but not in the virtualization model")
for vm in inconsistency_info['state_not_equal_to_desired_state']:
errors.append(f"{vm['id']} ({vm['email']}) is {vm['state']} but it is supposed to be {vm['desired_state']}")
for vm in inconsistency_info['stole_someone_elses_ip_and_own_ip_avaliable']:
errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_and_own_ip_avaliable current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
for vm in inconsistency_info['stole_someone_elses_ip_but_own_ip_also_stolen']:
errors.append(f"{vm['id']} ({vm['email']}) stole_someone_elses_ip_but_own_ip_also_stolen current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
for vm in inconsistency_info['has_wrong_ip']:
errors.append(f"{vm['id']} ({vm['email']}) has_wrong_ip current_ipv4={vm['current_ipv4']} desired_ipv4={vm['desired_ipv4']}")
if len(errors) > 0:
email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
for error in errors:
current_app.logger.info(f"cron_task: {error}.")
# current_app.config["FLASK_MAIL_INSTANCE"].send(
# Message(
# "Capsul Consistency Check Failed",
# sender=current_app.config["MAIL_DEFAULT_SENDER"],
# body="\n".join(errors),
# recipients=email_addresses
# )
# )
current_app.config["FLASK_MAIL_INSTANCE"].send(
Message(
"Capsul Consistency Check Failed",
sender=current_app.config["MAIL_DEFAULT_SENDER"],
body="\n".join(errors),
recipients=email_addresses
)
)

103
capsulflask/consistency.py

@ -32,18 +32,12 @@ def get_all_vms_from_db() -> dict:
vm['virtual_bridge_name'] = network['virtual_bridge_name']
vm['host'] = host_id
db_vms_by_id[vm['id']] = vm
# for vm in db_vms:
# if vm["id"] not in db_vms_by_id:
# # TODO
# raise Exception("non_deleted_vms_by_host_and_network did not return a vm that was returned by all_vm_ids_with_desired_state")
# else:
# db_vms_by_id[vm["id"]]["state"] = vm["desired_state"]
return db_vms_by_id
# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'
# this returns the same shape of object as get_all_vms_from_db except it has 'state' instead of 'desired_state'.
# also it has `macs` which is a list of strings
def get_all_vms_from_hosts(db_vms_by_id: dict) -> dict:
virt_vms_by_id = current_app.config["HUB_MODEL"].get_all_by_id()
#current_app.logger.info(f"\n*******************2:\n{pprint.pformat(virt_vms_by_id)}\n\n\n\n")
@ -68,46 +62,53 @@ def get_all_vms_from_hosts(db_vms_by_id: dict) -> dict:
return virt_vms_by_id
def ensure_vms_and_db_are_synced():
pass
# # Now creating the capsuls running status ui
# #
# for vm in db_vms:
# db_ids_dict[vm['id']] = vm['desired_state']
# for vm in virt_vms:
# virt_ids_dict[vm['id']] = vm['desired_state']
# errors = list()
# for id in db_ids_dict:
# if id not in virt_ids_dict:
# errors.append(f"{id} is in the database but not in the virtualization model")
# elif db_ids_dict[id] != virt_ids_dict[id]:
# errors.append(f"{id} has the desired state {db_ids_dict[id]} in the database but current state {virt_ids_dict[id]} in the virtualization model")
# for id in virt_ids_dict:
# if id not in db_ids_dict:
# errors.append(f"{id} is in the virtualization model but not in the database")
# if len(errors) > 0:
# email_addresses_raw = current_app.config['ADMIN_EMAIL_ADDRESSES'].split(",")
# email_addresses = list(filter(lambda x: len(x) > 6, map(lambda x: x.strip(), email_addresses_raw ) ))
# current_app.logger.info(f"cron_task: sending inconsistency warning email to {','.join(email_addresses)}:")
# for error in errors:
# current_app.logger.info(f"cron_task: {error}.")
# current_app.config["FLASK_MAIL_INSTANCE"].send(
# Message(
# "Capsul Consistency Check Failed",
# sender=current_app.config["MAIL_DEFAULT_SENDER"],
# body="\n".join(errors),
# recipients=email_addresses
# )
# )
def get_inconsistent_capsuls_information(db_vms_by_id: dict, virt_vms_by_id: dict) -> dict:
virt_vm_id_by_ipv4 = dict()
for vm_id, virt_vm in virt_vms_by_id.items():
if 'public_ipv4' in virt_vm and virt_vm['public_ipv4'] != "":
virt_vm_id_by_ipv4[virt_vm['public_ipv4']] = vm_id
db_vm_id_by_ipv4 = dict()
for vm_id, db_vm in db_vms_by_id.items():
if 'public_ipv4' in db_vm and db_vm['public_ipv4'] != "":
db_vm_id_by_ipv4[db_vm['public_ipv4']] = vm_id
in_db_but_not_in_virt = []
state_not_equal_to_desired_state = []
has_no_desired_ip_address = []
has_not_aquired_ip_address_yet = []
stole_someone_elses_ip_and_own_ip_avaliable = []
stole_someone_elses_ip_but_own_ip_also_stolen = []
has_wrong_ip = []
for vm_id, db_vm in db_vms_by_id.items():
if vm_id not in virt_vms_by_id:
in_db_but_not_in_virt.append(db_vm)
elif virt_vms_by_id[vm_id]['state'] != db_vm["desired_state"]:
db_vm["state"] = virt_vms_by_id[vm_id]['state']
state_not_equal_to_desired_state.append(db_vm)
elif 'public_ipv4' not in db_vm or db_vm["public_ipv4"] == "":
has_no_desired_ip_address.append(db_vm)
elif db_vm["desired_state"] == "running" and ('public_ipv4' not in virt_vms_by_id[vm_id] or virt_vms_by_id[vm_id]['public_ipv4'] == ""):
has_not_aquired_ip_address_yet.append(db_vm)
elif db_vm["desired_state"] == "running" and virt_vms_by_id[vm_id]['public_ipv4'] != db_vm["public_ipv4"]:
db_vm["desired_ipv4"] = db_vm["public_ipv4"]
db_vm["current_ipv4"] = virt_vms_by_id[vm_id]['public_ipv4']
if virt_vms_by_id[vm_id]['public_ipv4'] in db_vm_id_by_ipv4:
if db_vm["public_ipv4"] not in virt_vm_id_by_ipv4:
stole_someone_elses_ip_and_own_ip_avaliable.append(db_vm)
else:
stole_someone_elses_ip_but_own_ip_also_stolen.append(db_vm)
else:
has_wrong_ip.append(db_vm)
return {
"in_db_but_not_in_virt": in_db_but_not_in_virt,
"state_not_equal_to_desired_state": state_not_equal_to_desired_state,
"has_no_desired_ip_address": has_no_desired_ip_address,
"has_not_aquired_ip_address_yet": has_not_aquired_ip_address_yet,
"stole_someone_elses_ip_and_own_ip_avaliable": stole_someone_elses_ip_and_own_ip_avaliable,
"stole_someone_elses_ip_but_own_ip_also_stolen": stole_someone_elses_ip_but_own_ip_also_stolen,
"has_wrong_ip": has_wrong_ip,
}

Loading…
Cancel
Save