Browse Source

capacity limiter and general cleanup

forest-wip
forest 2 years ago
parent
commit
403506a0b0
  1. 14
      capsulflask/console.py
  2. 6
      capsulflask/metrics.py
  3. 3
      capsulflask/schema_migrations/02_up_accounts_vms_etc.sql
  4. 25
      capsulflask/shell_scripts/capacity-avaliable.sh
  5. 2
      capsulflask/shell_scripts/destroy.sh
  6. 2
      capsulflask/shell_scripts/get.sh
  7. 3
      capsulflask/shell_scripts/list-ids.sh
  8. 79
      capsulflask/templates/create-capsul.html
  9. 31
      capsulflask/virt_model.py
  10. 6
      setup.cfg

14
capsulflask/console.py

@ -75,7 +75,7 @@ def detail(id):
vm["ipv4"] = double_check_capsul_address(vm["id"], vm["ipv4"])
vm["created"] = vm['created'].strftime("%b %d %Y %H:%M")
vm["ssh_public_keys"] = ", ".join(vm["ssh_public_keys"]) if len(vm["ssh_public_keys"]) > 0 else "<deleted>"
vm["ssh_public_keys"] = ", ".join(vm["ssh_public_keys"]) if len(vm["ssh_public_keys"]) > 0 else "<missing>"
return render_template(
"capsul-detail.html",
@ -92,6 +92,7 @@ def create():
operating_systems = get_model().operating_systems_dict()
ssh_public_keys = get_model().list_ssh_public_keys_for_account(session["account"])
account_balance = get_account_balance()
capacity_avaliable = current_app.config["VIRTUALIZATION_MODEL"].capacity_avaliable(512*1024*1024)
errors = list()
created_os = None
@ -130,6 +131,13 @@ def create():
if len(posted_keys) == 0:
errors.append("At least one SSH Public Key is required")
capacity_avaliable = current_app.config["VIRTUALIZATION_MODEL"].capacity_avaliable(vm_sizes[size]['memory_mb']*1024*1024)
if not capacity_avaliable:
errors.append("""
host(s) at capacity. no capsuls can be created at this time. sorry.
""")
if len(errors) == 0:
id = makeCapsulId()
get_model().create_vm(
@ -157,9 +165,13 @@ def create():
for error in errors:
flash(error)
if not capacity_avaliable:
print(f"when capsul capacity is restored, send an email to {session['account']}")
return render_template(
"create-capsul.html",
created_os=created_os,
capacity_avaliable=capacity_avaliable,
account_balance=format(account_balance, '.2f'),
ssh_public_keys=ssh_public_keys,
ssh_public_key_count=len(ssh_public_keys),

6
capsulflask/metrics.py

@ -181,6 +181,7 @@ def draw_plot_png_bytes(data, scale, size_x=3, size_y=1):
my_plot.xaxis.set_major_formatter(day_formatter)
max_value = reduce(lambda a, b: a if a > b else b, y, scale)
average=(sum(y)/len(y))/scale
average=average*1.25+0.1
@ -195,11 +196,8 @@ def draw_plot_png_bytes(data, scale, size_x=3, size_y=1):
my_plot.fill_between( x, y, color=highlight_color, alpha=0.3)
my_plot.plot(x, y, 'r-', color=highlight_color)
my_plot.patch.set_facecolor('red')
my_plot.patch.set_alpha(0.5)
if size_y < 4:
my_plot.set_yticks([0, scale*0.5, scale])
my_plot.set_yticks([0, scale])
my_plot.set_ylim(0, scale)
my_plot.xaxis.label.set_color(highlight_color)

3
capsulflask/schema_migrations/02_up_accounts_vms_etc.sql

@ -71,7 +71,7 @@ CREATE TABLE stripe_checkout_sessions (
INSERT INTO os_images (id, template_image_file_name, description)
VALUES ('alpine311', 'alpine-cloud-2020-04-18.qcow2', 'Alpine Linux 3.11'),
('ubuntu18', 'ubuntu-18.04-minimal-cloudimg-amd64.img', 'Ubuntu 18.04 LTS (Bionic Beaver)'),
('ubuntu18', 'ubuntu-18.04-minimal-cloudimg-amd64.img', 'Ubuntu 18.04 LTS (Bionic)'),
('debian10', 'debian-10-genericcloud-amd64-20191117-80.qcow2', 'Debian 10 (Buster)'),
('centos7', 'CentOS-7-x86_64-GenericCloud.qcow2', 'CentOS 7'),
('centos8', 'CentOS-8-GenericCloud-8.1.1911-20200113.3.x86_64.qcow2', 'CentOS 8'),
@ -86,6 +86,7 @@ VALUES ('f1-s', 5.33, 512, 1, 500),
('f1-xx', 29.66, 8192, 4, 8000),
('f1-xxx', 57.58, 16384, 8, 16000);
-- this is test data to be removed later
INSERT INTO accounts (email)
VALUES ('forest.n.johnson@gmail.com');

25
capsulflask/shell_scripts/capacity-avaliable.sh

@ -0,0 +1,25 @@
#!/bin/sh
## check avaliable RAM and IPv4s
RAM_BYTES_TO_ALLOCATE="$1"
if echo "$RAM_BYTES_TO_ALLOCATE" | grep -vqE "^[0-9]+$"; then
echo "RAM_BYTES_TO_ALLOCATE \"$RAM_BYTES_TO_ALLOCATE\" must be an integer"
exit 1
fi
#100GB
RAM_LIMIT=100000000000
## compare current allocated ram + RAM_BYTES_TO_ALLOCATE with RAM_LIMIT
IPV4_LIMIT=26
IPV4_COUNT=$(grep ip-add /var/lib/libvirt/dnsmasq/virbr1.status | wc -l)
if [ $IPV4_COUNT -gt $IPV4_LIMIT ]; then
echo "IPv4 address limit reached"
exit 1
fi
echo "yes"

2
capsulflask/shell_scripts/destroy.sh

@ -10,7 +10,7 @@ three_dots() {
vmname="$1"
if echo "$vmname" | grep -vqE '^capsul-[a-z0-9]{10}$'; then
if echo "$vmname" | grep -vqE '^(cvm|capsul)-[a-z0-9]{10}$'; then
echo "vmname $vmname must match "'"^capsul-[a-z0-9]{10}$"'
exit 1
fi

2
capsulflask/shell_scripts/get.sh

@ -2,7 +2,7 @@
vmname="$1"
if echo "$vmname" | grep -vqE '^capsul-[a-z0-9]{10}$'; then
if echo "$vmname" | grep -vqE '^(cvm|capsul)-[a-z0-9]{10}$'; then
echo "vmname $vmname must match "'"^capsul-[a-z0-9]{10}$"'
exit 1
fi

3
capsulflask/shell_scripts/list-ids.sh

@ -0,0 +1,3 @@
#!/bin/sh
virsh list --all | grep running | grep -v ' Id' | grep -v -- '----' | awk '{print $2}' | sort

79
capsulflask/templates/create-capsul.html

@ -27,12 +27,13 @@
(Must be funded enough to last at least one month at creation time).
</p>
<p>You must <a href="/console/account-balance">add funds to your account</a> before you can create a Capsul.</p>
{% elif no_ssh_public_keys %}
<p>You don't have any ssh public keys yet.</p>
<p>You must <a href="/console/ssh">upload one</a> before you can create a Capsul.</p>
{% elif not capacity_avaliable %}
<p>Host(s) at capacity. No capsuls can be created at this time. sorry. </p>
{% else %}
{% if no_ssh_public_keys %}
<p>You don't have any ssh public keys yet.</p>
<p>You must <a href="/console/ssh">upload one</a> before you can create a Capsul.</p>
{% else %}
<pre>
<pre>
CAPSUL SIZES
============
type monthly cpus mem ssd net*
@ -46,44 +47,42 @@
* net is calculated as a per-month average
* all VMs come standard with one public IPv4 addr</pre>
<pre>
<pre>
Your <a href="/console/account-balance">account balance</a>: ${{ account_balance }}
</pre>
</pre>
<form method="post">
<div class="row justify-start">
<label class="align" for="size">Capsul Size</label>
<select id="size" name="size">
{% for size in vm_sizes.keys() %}<option value="{{ size }}">{{ size }}</option>{% endfor %}
</select>
<form method="post">
<div class="row justify-start">
<label class="align" for="size">Capsul Size</label>
<select id="size" name="size">
{% for size in vm_sizes.keys() %}<option value="{{ size }}">{{ size }}</option>{% endfor %}
</select>
</div>
<div class="row justify-start">
<label class="align" for="os">Operating System</label>
<select id="os" name="os">
{% for os_id, os in operating_systems.items() %}
<option value="{{ os_id }}">{{ os.description }}</option>
{% endfor %}
</select>
</div>
<div class="row justify-start">
<input type="hidden" name="ssh_public_key_count" value="{{ ssh_public_key_count}}"/>
<label class="align" for="ssh_keys">SSH Public Keys</label>
<div id="ssh_keys">
{% for key in ssh_public_keys %}
<label for="ssh_key_{{ loop.index - 1 }}">
<input type="checkbox" id="ssh_key_{{ loop.index - 1 }}" name="ssh_key_{{ loop.index - 1 }}" value="{{ key['name'] }}"/>
{{ key['name'] }}
</label>
{% endfor %}
</div>
<div class="row justify-start">
<label class="align" for="os">Operating System</label>
<select id="os" name="os">
{% for os_id, os in operating_systems.items() %}
<option value="{{ os_id }}">{{ os.description }}</option>
{% endfor %}
</select>
</div>
<div class="row justify-start">
<input type="hidden" name="ssh_public_key_count" value="{{ ssh_public_key_count}}"/>
<label class="align" for="ssh_keys">SSH Public Keys</label>
<div id="ssh_keys">
{% for key in ssh_public_keys %}
<label for="ssh_key_{{ loop.index - 1 }}">
<input type="checkbox" id="ssh_key_{{ loop.index - 1 }}" name="ssh_key_{{ loop.index - 1 }}" value="{{ key['name'] }}"/>
{{ key['name'] }}
</label>
{% endfor %}
</div>
</div>
<div class="row justify-end">
<input type="submit" value="Create">
</div>
</form>
</div>
{% endif %}
</div>
<div class="row justify-end">
<input type="submit" value="Create">
</div>
</form>
</div>
{% endif %}
{% endif %}

31
capsulflask/virt_model.py

@ -9,7 +9,7 @@ from subprocess import run
from capsulflask.db import get_model
def validate_capsul_id(id):
if not re.match(r"^capsul-[a-z0-9]{10}$", id):
if not re.match(r"^(cvm|capsul)-[a-z0-9]{10}$", id):
raise ValueError(f"vm id \"{id}\" must match \"^capsul-[a-z0-9]{{10}}$\"")
class VirtualMachine:
@ -19,6 +19,9 @@ class VirtualMachine:
self.ipv6 = ipv6
class VirtualizationInterface:
def capacity_avaliable(self, additional_ram_bytes: int) -> bool:
pass
def get(self, id: str) -> VirtualMachine:
pass
@ -32,6 +35,9 @@ class VirtualizationInterface:
pass
class MockVirtualization(VirtualizationInterface):
def capacity_avaliable(self, additional_ram_bytes):
return True
def get(self, id):
validate_capsul_id(id)
return VirtualMachine(id, ipv4="1.1.1.1")
@ -63,6 +69,27 @@ class ShellScriptVirtualization(VirtualizationInterface):
{completedProcess.stderr}
""")
def capacity_avaliable(self, additional_ram_bytes):
my_args=[join(current_app.root_path, 'shell_scripts/capacity-avaliable.sh'), additional_ram_bytes]
completedProcess = run(my_args, capture_output=True)
if completedProcess.returncode != 0:
print(f"""
capacity-avaliable.sh exited {completedProcess.returncode} with
stdout:
{completedProcess.stdout}
stderr:
{completedProcess.stderr}
""")
return False
lines = completedProcess.stdout.splitlines()
if not lines[len(lines)-1] == "yes":
print("capacity-avaliable.sh exited 0 but did not return \"yes\" ")
return False
return True
def get(self, id):
validate_capsul_id(id)
completedProcess = run([join(current_app.root_path, 'shell_scripts/get.sh'), id], capture_output=True)
@ -75,7 +102,7 @@ class ShellScriptVirtualization(VirtualizationInterface):
return VirtualMachine(id, ipv4=lines[0])
def list_ids(self) -> list:
completedProcess = run([join(current_app.root_path, 'shell_scripts/list_ids.sh')], capture_output=True)
completedProcess = run([join(current_app.root_path, 'shell_scripts/list-ids.sh')], capture_output=True)
self.validate_completed_process(completedProcess)
return completedProcess.stdout.splitlines()

6
setup.cfg

@ -14,4 +14,10 @@ packages = find:
include_package_data = true
install_requires =
Flask
Flask-Mail
psycopg2
nanoid
matplotlib
stripe
python-dotenv
requests

Loading…
Cancel
Save