From da698a36666589847fb387ea222a18e8a20dfbbd Mon Sep 17 00:00:00 2001 From: Lance Pioch Date: Tue, 21 May 2024 21:02:11 -0400 Subject: [PATCH 1/8] Remove exception --- .../Deployment/NoViableNodeException.php | 9 ------- .../Admin/Servers/CreateServerController.php | 1 - .../Nodes/NodeDeploymentController.php | 2 -- .../Application/Servers/ServerController.php | 1 - .../Deployment/FindViableNodesService.php | 25 +++---------------- .../Servers/ServerCreationService.php | 2 -- 6 files changed, 3 insertions(+), 37 deletions(-) delete mode 100644 app/Exceptions/Service/Deployment/NoViableNodeException.php diff --git a/app/Exceptions/Service/Deployment/NoViableNodeException.php b/app/Exceptions/Service/Deployment/NoViableNodeException.php deleted file mode 100644 index fc2fa47b8..000000000 --- a/app/Exceptions/Service/Deployment/NoViableNodeException.php +++ /dev/null @@ -1,9 +0,0 @@ -disk, 'Disk space must be an int, got %s'); Assert::integer($this->memory, 'Memory usage must be an int, got %s'); @@ -68,16 +59,6 @@ class FindViableNodesService ->havingRaw('(IFNULL(SUM(servers.memory), 0) + ?) <= (nodes.memory * (1 + (nodes.memory_overallocate / 100)))', [$this->memory]) ->havingRaw('(IFNULL(SUM(servers.disk), 0) + ?) <= (nodes.disk * (1 + (nodes.disk_overallocate / 100)))', [$this->disk]); - if (!is_null($page)) { - $results = $results->paginate($perPage ?? 50, ['*'], 'page', $page); - } else { - $results = $results->get()->toBase(); - } - - if ($results->isEmpty()) { - throw new NoViableNodeException(trans('exceptions.deployment.no_viable_nodes')); - } - - return $results; + return $results->get(); } } diff --git a/app/Services/Servers/ServerCreationService.php b/app/Services/Servers/ServerCreationService.php index 9d9603464..d1110ade4 100644 --- a/app/Services/Servers/ServerCreationService.php +++ b/app/Services/Servers/ServerCreationService.php @@ -42,7 +42,6 @@ class ServerCreationService * @throws \Throwable * @throws \App\Exceptions\DisplayException * @throws \Illuminate\Validation\ValidationException - * @throws \App\Exceptions\Service\Deployment\NoViableNodeException * @throws \App\Exceptions\Service\Deployment\NoViableAllocationException */ public function handle(array $data, DeploymentObject $deployment = null): Server @@ -105,7 +104,6 @@ class ServerCreationService * * @throws \App\Exceptions\DisplayException * @throws \App\Exceptions\Service\Deployment\NoViableAllocationException - * @throws \App\Exceptions\Service\Deployment\NoViableNodeException */ private function configureDeployment(array $data, DeploymentObject $deployment): Allocation { From f9aa8cf2186dc10973e1352be6a9bceb0bd7befd Mon Sep 17 00:00:00 2001 From: Lance Pioch Date: Tue, 21 May 2024 21:44:49 -0400 Subject: [PATCH 2/8] Simplify viable nodes service --- .../Nodes/NodeDeploymentController.php | 14 ++--- .../Deployment/FindViableNodesService.php | 54 +++++-------------- .../Servers/ServerCreationService.php | 12 +++-- .../Deployment/FindViableNodesServiceTest.php | 42 --------------- 4 files changed, 27 insertions(+), 95 deletions(-) delete mode 100644 tests/Integration/Services/Deployment/FindViableNodesServiceTest.php diff --git a/app/Http/Controllers/Api/Application/Nodes/NodeDeploymentController.php b/app/Http/Controllers/Api/Application/Nodes/NodeDeploymentController.php index 1f79bf674..b5ba80cf1 100644 --- a/app/Http/Controllers/Api/Application/Nodes/NodeDeploymentController.php +++ b/app/Http/Controllers/Api/Application/Nodes/NodeDeploymentController.php @@ -9,9 +9,6 @@ use App\Http\Requests\Api\Application\Nodes\GetDeployableNodesRequest; class NodeDeploymentController extends ApplicationApiController { - /** - * NodeDeploymentController constructor. - */ public function __construct(private FindViableNodesService $viableNodesService) { parent::__construct(); @@ -25,10 +22,13 @@ class NodeDeploymentController extends ApplicationApiController public function __invoke(GetDeployableNodesRequest $request): array { $data = $request->validated(); - $nodes = $this->viableNodesService - ->setMemory($data['memory']) - ->setDisk($data['disk']) - ->handle((int) $request->query('per_page'), (int) $request->query('page')); + + $nodes = $this->viableNodesService->handle( + $data['disk'] ?? 0, + $data['memory'] ?? 0, + $data['cpu'] ?? 0, + $data['location_ids'] ?? $data['tags'] ?? [], + ); return $this->fractal->collection($nodes) ->transformWith($this->getTransformer(NodeTransformer::class)) diff --git a/app/Services/Deployment/FindViableNodesService.php b/app/Services/Deployment/FindViableNodesService.php index 384f81aa0..37dff5fae 100644 --- a/app/Services/Deployment/FindViableNodesService.php +++ b/app/Services/Deployment/FindViableNodesService.php @@ -3,37 +3,10 @@ namespace App\Services\Deployment; use App\Models\Node; -use Webmozart\Assert\Assert; use Illuminate\Support\Collection; class FindViableNodesService { - protected ?int $disk = null; - protected ?int $memory = null; - - /** - * Set the amount of disk that will be used by the server being created. Nodes will be - * filtered out if they do not have enough available free disk space for this server - * to be placed on. - */ - public function setDisk(int $disk): self - { - $this->disk = $disk; - - return $this; - } - - /** - * Set the amount of memory that this server will be using. As with disk space, nodes that - * do not have enough free memory will be filtered out. - */ - public function setMemory(int $memory): self - { - $this->memory = $memory; - - return $this; - } - /** * Returns a collection of nodes that meet the provided requirements and can then * be passed to the AllocationSelectionService to return a single allocation. @@ -44,21 +17,20 @@ class FindViableNodesService * are tossed out, as are any nodes marked as non-public, meaning automatic * deployments should not be done against them. */ - public function handle(int $disk = null, int $memory = null, int $cpu = null): Collection + public function handle(int $disk = 0, int $memory = 0, int $cpu = 0, $tags = []): Collection { - Assert::integer($this->disk, 'Disk space must be an int, got %s'); - Assert::integer($this->memory, 'Memory usage must be an int, got %s'); + $nodes = Node::query() + ->withSum('servers', 'disk') + ->withSum('servers', 'memory') + ->withSum('servers', 'cpu') + ->where('public', true) + ->get(); - $query = Node::query()->select('nodes.*') - ->selectRaw('IFNULL(SUM(servers.memory), 0) as sum_memory') - ->selectRaw('IFNULL(SUM(servers.disk), 0) as sum_disk') - ->leftJoin('servers', 'servers.node_id', '=', 'nodes.id') - ->where('nodes.public', 1); - - $results = $query->groupBy('nodes.id') - ->havingRaw('(IFNULL(SUM(servers.memory), 0) + ?) <= (nodes.memory * (1 + (nodes.memory_overallocate / 100)))', [$this->memory]) - ->havingRaw('(IFNULL(SUM(servers.disk), 0) + ?) <= (nodes.disk * (1 + (nodes.disk_overallocate / 100)))', [$this->disk]); - - return $results->get(); + return $nodes + ->filter(fn (Node $node) => !$tags || collect($node->tags)->intersect($tags)) + ->filter(fn (Node $node) => $node->servers_sum_disk + $disk <= $node->disk * (1 + $node->disk_overallocate / 100)) + ->filter(fn (Node $node) => $node->servers_sum_memory + $memory <= $node->memory * (1 + $node->memory_overallocate / 100)) + ->filter(fn (Node $node) => $node->servers_sum_cpu + $cpu <= $node->cpu * (1 + $node->cpu_overallocate / 100)) + ; } } diff --git a/app/Services/Servers/ServerCreationService.php b/app/Services/Servers/ServerCreationService.php index d1110ade4..8953b1325 100644 --- a/app/Services/Servers/ServerCreationService.php +++ b/app/Services/Servers/ServerCreationService.php @@ -107,11 +107,13 @@ class ServerCreationService */ private function configureDeployment(array $data, DeploymentObject $deployment): Allocation { - /** @var \Illuminate\Support\Collection $nodes */ - $nodes = $this->findViableNodesService - ->setDisk(Arr::get($data, 'disk')) - ->setMemory(Arr::get($data, 'memory')) - ->handle(); + /** @var Collection<\App\Models\Node> $nodes */ + $nodes = $this->findViableNodesService->handle( + Arr::get($data, 'disk', 0), + Arr::get($data, 'memory', 0), + Arr::get($data, 'cpu', 0), + Arr::get($data, 'tags', []), + ); return $this->allocationSelectionService->setDedicated($deployment->isDedicated()) ->setNodes($nodes->pluck('id')->toArray()) diff --git a/tests/Integration/Services/Deployment/FindViableNodesServiceTest.php b/tests/Integration/Services/Deployment/FindViableNodesServiceTest.php deleted file mode 100644 index d9a94cbc9..000000000 --- a/tests/Integration/Services/Deployment/FindViableNodesServiceTest.php +++ /dev/null @@ -1,42 +0,0 @@ -delete(); - Server::query()->delete(); - Node::query()->delete(); - } - - public function testExceptionIsThrownIfNoDiskSpaceHasBeenSet(): void - { - $this->expectException(\InvalidArgumentException::class); - $this->expectExceptionMessage('Disk space must be an int, got NULL'); - - $this->getService()->handle(); - } - - public function testExceptionIsThrownIfNoMemoryHasBeenSet(): void - { - $this->expectException(\InvalidArgumentException::class); - $this->expectExceptionMessage('Memory usage must be an int, got NULL'); - - $this->getService()->setDisk(10)->handle(); - } - - private function getService(): FindViableNodesService - { - return $this->app->make(FindViableNodesService::class); - } -} From 5c33c7495a2af651917d2b8dc22010e96a2376a7 Mon Sep 17 00:00:00 2001 From: Lance Pioch Date: Tue, 21 May 2024 21:45:26 -0400 Subject: [PATCH 3/8] Ignore this for now --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 0a2c37f0a..000336813 100644 --- a/.gitignore +++ b/.gitignore @@ -39,3 +39,4 @@ docker-compose.yaml public/css/filament-monaco-editor/ public/js/filament-monaco-editor/ +/database/testing From b579f14f3f577d209abc2eb74dad4f23d2e8b25a Mon Sep 17 00:00:00 2001 From: Lance Pioch Date: Tue, 21 May 2024 22:04:12 -0400 Subject: [PATCH 4/8] Backwards compatibility --- .../Nodes/NodeDeploymentController.php | 2 +- .../Nodes/GetDeployableNodesRequest.php | 2 ++ .../Application/Servers/StoreServerRequest.php | 10 ++++++++++ app/Models/Objects/DeploymentObject.php | 15 +++++++++++++++ 4 files changed, 28 insertions(+), 1 deletion(-) diff --git a/app/Http/Controllers/Api/Application/Nodes/NodeDeploymentController.php b/app/Http/Controllers/Api/Application/Nodes/NodeDeploymentController.php index b5ba80cf1..f00c8d667 100644 --- a/app/Http/Controllers/Api/Application/Nodes/NodeDeploymentController.php +++ b/app/Http/Controllers/Api/Application/Nodes/NodeDeploymentController.php @@ -27,7 +27,7 @@ class NodeDeploymentController extends ApplicationApiController $data['disk'] ?? 0, $data['memory'] ?? 0, $data['cpu'] ?? 0, - $data['location_ids'] ?? $data['tags'] ?? [], + $data['tags'] ?? $data['location_ids'] ?? [], ); return $this->fractal->collection($nodes) diff --git a/app/Http/Requests/Api/Application/Nodes/GetDeployableNodesRequest.php b/app/Http/Requests/Api/Application/Nodes/GetDeployableNodesRequest.php index c5c351cf1..e421be5d4 100644 --- a/app/Http/Requests/Api/Application/Nodes/GetDeployableNodesRequest.php +++ b/app/Http/Requests/Api/Application/Nodes/GetDeployableNodesRequest.php @@ -10,6 +10,8 @@ class GetDeployableNodesRequest extends GetNodesRequest 'page' => 'integer', 'memory' => 'required|integer|min:0', 'disk' => 'required|integer|min:0', + 'location_ids' => 'array', + 'tags' => 'array', ]; } } diff --git a/app/Http/Requests/Api/Application/Servers/StoreServerRequest.php b/app/Http/Requests/Api/Application/Servers/StoreServerRequest.php index d3343e7a5..fd735edf0 100644 --- a/app/Http/Requests/Api/Application/Servers/StoreServerRequest.php +++ b/app/Http/Requests/Api/Application/Servers/StoreServerRequest.php @@ -123,6 +123,15 @@ class StoreServerRequest extends ApplicationApiRequest return !$input->deploy; }); + /** @deprecated use tags instead */ + $validator->sometimes('deploy.locations', 'present', function ($input) { + return $input->deploy; + }); + + $validator->sometimes('deploy.tags', 'present', function ($input) { + return $input->deploy; + }); + $validator->sometimes('deploy.port_range', 'present', function ($input) { return $input->deploy; }); @@ -139,6 +148,7 @@ class StoreServerRequest extends ApplicationApiRequest $object = new DeploymentObject(); $object->setDedicated($this->input('deploy.dedicated_ip', false)); + $object->setTags($this->input('deploy.tags', $this->input('deploy.locations', []))); $object->setPorts($this->input('deploy.port_range', [])); return $object; diff --git a/app/Models/Objects/DeploymentObject.php b/app/Models/Objects/DeploymentObject.php index d5b02dd83..b7a4ebaf3 100644 --- a/app/Models/Objects/DeploymentObject.php +++ b/app/Models/Objects/DeploymentObject.php @@ -6,6 +6,8 @@ class DeploymentObject { private bool $dedicated = false; + private array $tags = []; + private array $ports = []; public function isDedicated(): bool @@ -31,4 +33,17 @@ class DeploymentObject return $this; } + + public function getTags(): array + { + return $this->tags; + } + + public function setTags(array $tags): self + { + $this->tags = $tags; + + return $this; + } + } From b007e63937645d9651f9eadcfa7830957ed8160f Mon Sep 17 00:00:00 2001 From: Lance Pioch Date: Wed, 22 May 2024 02:35:20 -0400 Subject: [PATCH 5/8] Pint fixes --- app/Services/Deployment/FindViableNodesService.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/Services/Deployment/FindViableNodesService.php b/app/Services/Deployment/FindViableNodesService.php index 37dff5fae..cffc64990 100644 --- a/app/Services/Deployment/FindViableNodesService.php +++ b/app/Services/Deployment/FindViableNodesService.php @@ -30,7 +30,6 @@ class FindViableNodesService ->filter(fn (Node $node) => !$tags || collect($node->tags)->intersect($tags)) ->filter(fn (Node $node) => $node->servers_sum_disk + $disk <= $node->disk * (1 + $node->disk_overallocate / 100)) ->filter(fn (Node $node) => $node->servers_sum_memory + $memory <= $node->memory * (1 + $node->memory_overallocate / 100)) - ->filter(fn (Node $node) => $node->servers_sum_cpu + $cpu <= $node->cpu * (1 + $node->cpu_overallocate / 100)) - ; + ->filter(fn (Node $node) => $node->servers_sum_cpu + $cpu <= $node->cpu * (1 + $node->cpu_overallocate / 100)); } } From 48f715ae699dc30273c678988bb04002bab96052 Mon Sep 17 00:00:00 2001 From: Lance Pioch Date: Wed, 22 May 2024 02:52:49 -0400 Subject: [PATCH 6/8] Fix directory --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 2d7eaeb46..bcc56c1a7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -99,7 +99,7 @@ jobs: QUEUE_CONNECTION: sync HASHIDS_SALT: alittlebitofsalt1234 DB_CONNECTION: sqlite - DB_DATABASE: ${{ github.workspace }}/database/testing.sqlite + DB_DATABASE: testing.sqlite steps: - name: Code Checkout uses: actions/checkout@v4 From 5b9e4b172989709676762727025d4a0f99db36f1 Mon Sep 17 00:00:00 2001 From: Lance Pioch Date: Wed, 22 May 2024 03:10:33 -0400 Subject: [PATCH 7/8] Always limit by nodes, was like this before anyways --- app/Services/Deployment/AllocationSelectionService.php | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/app/Services/Deployment/AllocationSelectionService.php b/app/Services/Deployment/AllocationSelectionService.php index 54853c025..715533cf5 100644 --- a/app/Services/Deployment/AllocationSelectionService.php +++ b/app/Services/Deployment/AllocationSelectionService.php @@ -90,11 +90,10 @@ class AllocationSelectionService */ private function getRandomAllocation(array $nodes = [], array $ports = [], bool $dedicated = false): ?Allocation { - $query = Allocation::query()->whereNull('server_id'); - - if (!empty($nodes)) { - $query->whereIn('node_id', $nodes); - } + $query = Allocation::query() + ->whereNull('server_id') + ->whereIn('node_id', $nodes) + ; if (!empty($ports)) { $query->where(function ($inner) use ($ports) { From f4672c6cb1c4b5932a993a8106a6aba862a7c4df Mon Sep 17 00:00:00 2001 From: Lance Pioch Date: Wed, 22 May 2024 03:15:29 -0400 Subject: [PATCH 8/8] Pint fix --- app/Services/Deployment/AllocationSelectionService.php | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/Services/Deployment/AllocationSelectionService.php b/app/Services/Deployment/AllocationSelectionService.php index 715533cf5..3868d0b38 100644 --- a/app/Services/Deployment/AllocationSelectionService.php +++ b/app/Services/Deployment/AllocationSelectionService.php @@ -92,8 +92,7 @@ class AllocationSelectionService { $query = Allocation::query() ->whereNull('server_id') - ->whereIn('node_id', $nodes) - ; + ->whereIn('node_id', $nodes); if (!empty($ports)) { $query->where(function ($inner) use ($ports) {