From dbfc18ed3cef55a4433870ddee77c921b996cc2b Mon Sep 17 00:00:00 2001 From: Matthieu Date: Fri, 8 Oct 2021 14:40:29 +0400 Subject: [PATCH] docs: add docusaurus base --- LICENSE | 201 ------------------ TODO.md | 14 -- .../rpc/nova.management.rpc.v1alpha.proto | 4 + docs/README.md | 1 - docs/common.md | 6 - docs/components/auto-scaler.md | 0 docs/components/cache.md | 0 docs/components/gateway.md | 0 docs/components/ratelimiter.md | 0 docs/components/webhook.md | 8 - docs/docs/LICENSE.md | 201 ++++++++++++++++++ docs/docs/deployment/_category_.json | 4 + docs/docs/deployment/production.md | 1 + docs/docs/internals/_category_.json | 4 + .../docs/internals/components/_category_.json | 4 + docs/docs/internals/components/gateway.md | 34 +++ docs/docs/internals/index.md | 51 +++++ docs/docs/internals/protocols/index.md | 7 + docs/docs/intro.md | 29 +++ docs/docs/quickstart.md | 21 ++ docs/exemples/config.yaml | 0 docs/internals/management.md | 5 - docs/structure.md | 15 -- manager/BUILD.bazel | 19 -- manager/lib/BUILD.bazel | 16 -- manager/lib/config.go | 10 - manager/lib/main.go | 27 --- manager/lib/scheduler/BUILD.bazel | 12 -- manager/lib/scheduler/main.go | 100 --------- manager/lib/services/BUILD.bazel | 9 - manager/lib/services/status-server.go | 26 --- manager/main.go | 26 --- ....v1.proto => nova.ratelimit.v1alpha.proto} | 76 +++---- webhook/config/default.yaml | 13 -- 34 files changed, 398 insertions(+), 546 deletions(-) delete mode 100644 LICENSE delete mode 100644 TODO.md create mode 100644 common/management/rpc/nova.management.rpc.v1alpha.proto delete mode 100644 docs/README.md delete mode 100644 docs/common.md delete mode 100644 docs/components/auto-scaler.md delete mode 100644 docs/components/cache.md delete mode 100644 docs/components/gateway.md delete mode 100644 docs/components/ratelimiter.md delete mode 100644 docs/components/webhook.md create mode 100644 docs/docs/LICENSE.md create mode 100644 docs/docs/deployment/_category_.json create mode 100644 docs/docs/deployment/production.md create mode 100644 docs/docs/internals/_category_.json create mode 100644 docs/docs/internals/components/_category_.json create mode 100644 docs/docs/internals/components/gateway.md create mode 100644 docs/docs/internals/index.md create mode 100644 docs/docs/internals/protocols/index.md create mode 100644 docs/docs/intro.md create mode 100644 docs/docs/quickstart.md delete mode 100644 docs/exemples/config.yaml delete mode 100644 docs/internals/management.md delete mode 100644 docs/structure.md delete mode 100644 manager/BUILD.bazel delete mode 100644 manager/lib/BUILD.bazel delete mode 100644 manager/lib/config.go delete mode 100644 manager/lib/main.go delete mode 100644 manager/lib/scheduler/BUILD.bazel delete mode 100644 manager/lib/scheduler/main.go delete mode 100644 manager/lib/services/BUILD.bazel delete mode 100644 manager/lib/services/status-server.go delete mode 100644 manager/main.go rename ratelimiter/proto/{nova.ratelimit.v1.proto => nova.ratelimit.v1alpha.proto} (96%) delete mode 100644 webhook/config/default.yaml diff --git a/LICENSE b/LICENSE deleted file mode 100644 index 9284bc7..0000000 --- a/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2021 Developer's House - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/TODO.md b/TODO.md deleted file mode 100644 index 6d56cdf..0000000 --- a/TODO.md +++ /dev/null @@ -1,14 +0,0 @@ -# Todo - -## GitHub - -- PR and Issue template -- Contributing GUIDE - -### Readme - -- Features -- Explaination of each components -- How to use (dev, prod) -- Contributing -- Sponsor? \ No newline at end of file diff --git a/common/management/rpc/nova.management.rpc.v1alpha.proto b/common/management/rpc/nova.management.rpc.v1alpha.proto new file mode 100644 index 0000000..e3b5c96 --- /dev/null +++ b/common/management/rpc/nova.management.rpc.v1alpha.proto @@ -0,0 +1,4 @@ +syntax = "proto3"; + +import "common/management/nova.management.v1alpha.proto"; +package nova.management.rpc.v1alpha; \ No newline at end of file diff --git a/docs/README.md b/docs/README.md deleted file mode 100644 index fcc815a..0000000 --- a/docs/README.md +++ /dev/null @@ -1 +0,0 @@ -# Nova documentation \ No newline at end of file diff --git a/docs/common.md b/docs/common.md deleted file mode 100644 index 3dce3ac..0000000 --- a/docs/common.md +++ /dev/null @@ -1,6 +0,0 @@ -# Common packages - -Nova exposes common packages for all the different projects in the mono-repo. -Since Nova uses two different languages, it has two different common libraries, -one built in rust in [the common/rust](../common/rust) directory and one in Go -in [the common/](../common) directory. \ No newline at end of file diff --git a/docs/components/auto-scaler.md b/docs/components/auto-scaler.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/components/cache.md b/docs/components/cache.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/components/gateway.md b/docs/components/gateway.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/components/ratelimiter.md b/docs/components/ratelimiter.md deleted file mode 100644 index e69de29..0000000 diff --git a/docs/components/webhook.md b/docs/components/webhook.md deleted file mode 100644 index 76361ef..0000000 --- a/docs/components/webhook.md +++ /dev/null @@ -1,8 +0,0 @@ -# Webhook - -> TD;TR The webhook component is an implementation of the discord interaction webhooks; You can either use the Gateway or the Webhooks. The webhooks __require__ an external https endpoint to work. - -The webhook source code is located in the [webhook](../../webhook) folder and is implemented in Rust. It's a simple http web server which implements the webhook signature verification and deserialization. Like the gateway, the messages are redirected using the [relaying system](../common#relaying_trait). - -The signature verification is done using libsodium via the libsodium-sys trait. -Subsequently, it uses code marked as "unsafe" in rust. It's built into the binary statically. Any route can be used to receive webhook messages. \ No newline at end of file diff --git a/docs/docs/LICENSE.md b/docs/docs/LICENSE.md new file mode 100644 index 0000000..7342771 --- /dev/null +++ b/docs/docs/LICENSE.md @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright 2021 Developer's House + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/docs/docs/deployment/_category_.json b/docs/docs/deployment/_category_.json new file mode 100644 index 0000000..27fb1fc --- /dev/null +++ b/docs/docs/deployment/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Deployment", + "position": 3 +} \ No newline at end of file diff --git a/docs/docs/deployment/production.md b/docs/docs/deployment/production.md new file mode 100644 index 0000000..38ca6b8 --- /dev/null +++ b/docs/docs/deployment/production.md @@ -0,0 +1 @@ +# Production guide \ No newline at end of file diff --git a/docs/docs/internals/_category_.json b/docs/docs/internals/_category_.json new file mode 100644 index 0000000..ed12ab8 --- /dev/null +++ b/docs/docs/internals/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Internals", + "position": 99999 +} \ No newline at end of file diff --git a/docs/docs/internals/components/_category_.json b/docs/docs/internals/components/_category_.json new file mode 100644 index 0000000..a50cc69 --- /dev/null +++ b/docs/docs/internals/components/_category_.json @@ -0,0 +1,4 @@ +{ + "label": "Components", + "position": 4 +} \ No newline at end of file diff --git a/docs/docs/internals/components/gateway.md b/docs/docs/internals/components/gateway.md new file mode 100644 index 0000000..9aab5c2 --- /dev/null +++ b/docs/docs/internals/components/gateway.md @@ -0,0 +1,34 @@ +--- +title: Gateway +position: 1 +--- + +The gateway if the component responsible for the real-time events +using the Discord Gateway API (websocket), the component is written +in rust using [`tokio-tungstenite`](https://github.com/snapview/tokio-tungstenite) and his job is to simply maintain +a gateway connection (heartbeat, reconnect, etc...) and send events +directly to the cache using nats. + +The project is located in the `gateway` folder at the root +of the GitHub repo and is published using a statically linked +library (except libc) and docker images. + +## Structure + +Internally, the gateway is divided in multiple rust modules + +### Connection (src/connection) + +This module implements a [`futures_core::stream::Stream` trait](https://docs.rs/futures-core/0.3.17/futures_core/stream/trait.Stream.html) +and [`futures::sink::Sink` trait](https://docs.rs/futures/0.3.17/futures/sink/trait.Sink.html) that streams deserialized discord +packets to the stream. It does not implement any reconnect mechanism and returns +an error when the connection encounters a deserialization error or i/o error. +If the connection is closed, the stream simply closes. + +You can send any `BaseMessage` struct through the connection and receive +any `BaseMessage` struct. + + +### Payloads (src/payload) + +This module implements the deserialization of discord packets diff --git a/docs/docs/internals/index.md b/docs/docs/internals/index.md new file mode 100644 index 0000000..d2bbd9f --- /dev/null +++ b/docs/docs/internals/index.md @@ -0,0 +1,51 @@ +--- +sidebar_position: 1 +--- + +# Nova Internals + +## Definitions + +### Cache + +A cache is an instance of the `cache` program, it managed everything that is related to data-management +of the discord events, webhooks and more. + +### Gateway +*todo* +### Webhook +*todo* +### + +## Communication + +``` + ┌───────────────┐ ┌───────────────────┐ + │ Gateway │ event(s) │ │ ┌───────────────────────┐ +┌──►│ (gateway) ├─────────────►│ ├──────────────────►│ │ +│ └───────────────┘ (nats queue) │ Cache instance │ (redis channel) │ Caching server │ +│ │ (cache) │ │ (redis) │ +│ ┌───────────────┐ event(s) │ │◄──────────────────┤ │ +│ │ Webhook ├─────────────►│ │ └───────────────────────┘ +│ │ (webhook) │ (nats queue) └─────────────────┬─┘ +│ └───────────────┘ ▲ │ +│ cache requests │ │ event(s) +│ (grpc) │ │ (nats queue) +│ │ ▼ +│ ┌───┴───────────────┐ +└──────────────────────────────────┤ User programs │ + gateway commands │ (client libs) │ + (grpc) └───────────────────┘ +``` + +> We try to use protocol buffers everywhere even inside the event broker (nats), +> this ensures we have the same date representation across all the programs written +> in different languages + +> The only dependency of the nova architecture is a reliable nats & redis cluster for now + +#### Protocols + +All the protocols a detailed in the [protocols folder](protocols) + +#### \ No newline at end of file diff --git a/docs/docs/internals/protocols/index.md b/docs/docs/internals/protocols/index.md new file mode 100644 index 0000000..4de3a69 --- /dev/null +++ b/docs/docs/internals/protocols/index.md @@ -0,0 +1,7 @@ +# Protocols + +This section of the internal documentation describes the protocols used by nova +to communicate between all the components. + + +* \ No newline at end of file diff --git a/docs/docs/intro.md b/docs/docs/intro.md new file mode 100644 index 0000000..451d2ab --- /dev/null +++ b/docs/docs/intro.md @@ -0,0 +1,29 @@ +# Introduction + +## What is nova ? + +Nova is a complete framework for building reliable and stable discord bots +using a services based system to operate all the components needed to operate +a discord such as the discord gateway, new discord webhooks for receiving interactions. +Using a traditional infrastructure (sharder / process), we can quickly reach bottlenecks +when we scale to multiple thousands of shards. Nova is a way to avoid these problems by +building a scale-first discord framework that creates an environment that allow +better testing, reliability and operations easier. + +### Advantages + +With the help of Nova, you can achieve a number of things, such as + +* Scaling of workers independent of the number of shards +* zero-login updates +* Automatic shards scaling +* Shared cache for the whole bot (without broadcastEval and other unsafe methods) +* Stateless workers, easier to test +* Distributed rest rate-limiting +* Easier fine-tuned monitoring using cloud-native technologies +* Languages agnostic apis + +### How did we solve this ? + +Nova separates the gateway into multiple smaller components corresponding to multiple +discord apis diff --git a/docs/docs/quickstart.md b/docs/docs/quickstart.md new file mode 100644 index 0000000..216552e --- /dev/null +++ b/docs/docs/quickstart.md @@ -0,0 +1,21 @@ +--- +sidebar_position: 2 +--- + +# 5 Minutes quickstart + +This page shows you how to start a new nova project +under five minutes using a typescript project, +hold tight this is going to be fast. + +## Requirements + +* A discord bot application available +* [Docker](https://docker.io/) (or alternatives) available to you. +* A domain name / [ngrok.io](https://ngrok.com/) domain (for webhooks only) + +> If you are deploying nova to production, consider following the +> production guide instead. + +## Setting up a nova instance + diff --git a/docs/exemples/config.yaml b/docs/exemples/config.yaml deleted file mode 100644 index e69de29..0000000 diff --git a/docs/internals/management.md b/docs/internals/management.md deleted file mode 100644 index 7bff71b..0000000 --- a/docs/internals/management.md +++ /dev/null @@ -1,5 +0,0 @@ -# Nova Internals - Management - -In order to run in multiple environments, Nova uses a scheduling system to balance shards between "cluster" instances. -Each cluster instance is in fact a gateway instance. All the management is handled in the "manager" component. -The discord shards are dynamically scaled and restarted if needed. \ No newline at end of file diff --git a/docs/structure.md b/docs/structure.md deleted file mode 100644 index 1f82ae3..0000000 --- a/docs/structure.md +++ /dev/null @@ -1,15 +0,0 @@ -# Workspace/System structure - -## System structure - -In the nova repository, two different types of projects exist, - -* The management projects primarly in Go \ - They manage the other components of the nova infrastructure. -* The data-path projects \ - They handle all the data transfer / management. - -### Gateway - -> The gateway interfaces with the discord gateway to retrive events in real time -It's implemented in rust and is in the gateway folder. \ No newline at end of file diff --git a/manager/BUILD.bazel b/manager/BUILD.bazel deleted file mode 100644 index e82b5ed..0000000 --- a/manager/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") - -go_library( - name = "manager_lib", - srcs = ["main.go"], - importpath = "github.com/discordnova/nova/manager", - visibility = ["//visibility:private"], - deps = [ - "//manager/lib", - "//manager/lib/scheduler", - "@com_github_buraksezer_consistent//:consistent", - ], -) - -go_binary( - name = "manager", - embed = [":manager_lib"], - visibility = ["//visibility:public"], -) diff --git a/manager/lib/BUILD.bazel b/manager/lib/BUILD.bazel deleted file mode 100644 index 9e3be05..0000000 --- a/manager/lib/BUILD.bazel +++ /dev/null @@ -1,16 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "lib", - srcs = [ - "config.go", - "main.go", - ], - importpath = "github.com/discordnova/nova/manager/lib", - visibility = ["//visibility:public"], - deps = [ - "//common/management", - "//manager/lib/services", - "@org_golang_google_grpc//:go_default_library", - ], -) diff --git a/manager/lib/config.go b/manager/lib/config.go deleted file mode 100644 index 07d0de9..0000000 --- a/manager/lib/config.go +++ /dev/null @@ -1,10 +0,0 @@ -package lib - - -type GrpcConfig struct { - Server string -} - -type Config struct { - Grpc GrpcConfig -} diff --git a/manager/lib/main.go b/manager/lib/main.go deleted file mode 100644 index 554b3fb..0000000 --- a/manager/lib/main.go +++ /dev/null @@ -1,27 +0,0 @@ -package lib - -import ( - "github.com/discordnova/nova/common/management" - "github.com/discordnova/nova/manager/lib/services" - "google.golang.org/grpc" - "log" - "net" -) - -func StartGrpcServices(config *Config) { - lis, err := net.Listen("tcp", config.Grpc.Server) - - if err != nil { - log.Fatalf("failed to start the grpc services: %v", err) - } - var opts []grpc.ServerOption - - grpcServer := grpc.NewServer(opts...) - - management.RegisterManagementServiceServer(grpcServer, services.CreateStatusServerService()) - - err = grpcServer.Serve(lis) - if err != nil { - log.Fatalf("failed to start the grpc server: %v", err) - } -} diff --git a/manager/lib/scheduler/BUILD.bazel b/manager/lib/scheduler/BUILD.bazel deleted file mode 100644 index 1c202b1..0000000 --- a/manager/lib/scheduler/BUILD.bazel +++ /dev/null @@ -1,12 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "scheduler", - srcs = ["main.go"], - importpath = "github.com/discordnova/nova/manager/lib/scheduler", - visibility = ["//visibility:public"], - deps = [ - "@com_github_buraksezer_consistent//:consistent", - "@com_github_cespare_xxhash//:xxhash", - ], -) diff --git a/manager/lib/scheduler/main.go b/manager/lib/scheduler/main.go deleted file mode 100644 index ef9c3b6..0000000 --- a/manager/lib/scheduler/main.go +++ /dev/null @@ -1,100 +0,0 @@ -package scheduler - -import ( - "github.com/buraksezer/consistent" - "github.com/cespare/xxhash" - "log" -) - -// consistent package doesn't provide a default hashing function. -// You should provide a proper one to distribute keys/members uniformly. -type hasher struct{} - -func (h hasher) Sum64(data []byte) uint64 { - // you should use a proper hash function for uniformity. - return xxhash.Sum64(data) -} - -type State struct { - Members []consistent.Member - CurrentSize int - MaxConcurrency int -} - -type Scheduler struct { - state State - hashRing *consistent.Consistent -} - -func CreateScheduler(state State) *Scheduler { - config := consistent.Config{ - Hasher: hasher{}, - PartitionCount: 271, - ReplicationFactor: 20, - Load: 1.25, - } - - return &Scheduler{ - state: state, - hashRing: consistent.New(state.Members, config), - } -} - -// Scale changes the size of Discord shards -func (s *Scheduler) Scale(newScale int) { - log.Printf("scaling from %d shards to %d shards", s.state.CurrentSize, newScale) - // todo: take the scale mutex - - // we should handle all the sessions restart - // and the re-scheduling of the shards including rate-limiting - // first, we remove all the shards instances from the clusters - for i := 0; i < s.state.CurrentSize; i++ { - // todo: call the rpc for stopping the shard on the given cluster - cluster := s.hashRing.LocateKey([]byte{byte(i)}) - log.Printf("sending stop shard to shard %d on cluster %s", i, cluster) - } - s.state.CurrentSize = newScale - - - // after we stopped the shard, we re-start it with the new number of shards - // since all the shard must be restarted after a scale operation - // but before that, we need to allocate all the shards - for i := 0; i < s.state.CurrentSize; i++ { - cluster := s.hashRing.LocateKey([]byte{byte(i)}) - // todo: call the rpc for allocate the shard on the given cluster - log.Printf("sending allocate shard to shard %d on %s with [%d, %d] as sharding key", i, cluster, i, newScale) - } - - // we start the first shard, this represents the "pilot" shard which gives us the - // required information, such as the max_concurrency. - // todo: call the rpc to start the first shard - - // according to the max_concurrency system, we must - // start the shards sequentially. - for i := 0; i < s.state.MaxConcurrency; i++ { - log.Printf("starting bucket %d", i) - for j := 0; j < newScale; j++ { - // if the shard corresponds to the current starting bucket - if j % s.state.MaxConcurrency == i { - cluster := s.hashRing.LocateKey([]byte{byte(j)}) - // todo: call the start rpc on the given cluster - log.Printf("%d: sending the start command to the shard %d on the %s cluster", i, j, cluster) - } - } - } - - // todo: release the scale mutex -} - -// RemoveNode removes a cluster from the consistent hash ring and relocate the shards -func (s *Scheduler) RemoveNode() { - -} - -// AddNode adds a node to the consistent hash ring and relocate the shards -func (s *Scheduler) AddNode() {} - -// GetShardLocation returns the supposed name of the cluster containing a Shard -func (s *Scheduler) GetShardLocation() string { - return "" -} diff --git a/manager/lib/services/BUILD.bazel b/manager/lib/services/BUILD.bazel deleted file mode 100644 index 5bb70d6..0000000 --- a/manager/lib/services/BUILD.bazel +++ /dev/null @@ -1,9 +0,0 @@ -load("@io_bazel_rules_go//go:def.bzl", "go_library") - -go_library( - name = "services", - srcs = ["status-server.go"], - importpath = "github.com/discordnova/nova/manager/lib/services", - visibility = ["//visibility:public"], - deps = ["//common/management"], -) diff --git a/manager/lib/services/status-server.go b/manager/lib/services/status-server.go deleted file mode 100644 index 2634edc..0000000 --- a/manager/lib/services/status-server.go +++ /dev/null @@ -1,26 +0,0 @@ -package services - -import ( - "context" - "fmt" - "github.com/discordnova/nova/common/management" -) - -type statusServer struct { - management.UnimplementedManagementServiceServer -} - -func CreateStatusServerService() management.ManagementServiceServer { - return &statusServer{} -} - -func (s *statusServer) GetGlobalClusterStatus(context context.Context, _ *management.Empty) (*management.GlobalClusterStatusResponse, error) { - return nil, fmt.Errorf("not implemented") -} -func (s *statusServer) GetClusterStatus(context context.Context, request *management.ClusterStatusRequest) (*management.ClusterStatusResponse, error) { - return nil, fmt.Errorf("not implemented") -} - -func (s *statusServer) GetShardStatus(context context.Context, request *management.ShardStatusRequest) (*management.ShardStatusResponse, error) { - return nil, fmt.Errorf("not implemented") -} diff --git a/manager/main.go b/manager/main.go deleted file mode 100644 index 296a25b..0000000 --- a/manager/main.go +++ /dev/null @@ -1,26 +0,0 @@ -package main - -import ( - "github.com/buraksezer/consistent" - "github.com/discordnova/nova/manager/lib" - "github.com/discordnova/nova/manager/lib/scheduler" - "log" -) - -type myMember string - -func (m myMember) String() string { - return string(m) -} - -func main () { - scheduler.CreateScheduler(scheduler.State{ - Members: []consistent.Member{myMember("cluster1"), myMember("cluster2"), myMember("cluster3"), myMember("cluster4")}, - CurrentSize: 2, - MaxConcurrency: 5, - }).Scale(10) - log.Printf("starting grpc") - lib.StartGrpcServices(&lib.Config{ - Grpc: lib.GrpcConfig{Server: "0.0.0.0:8053"}, - }) -} \ No newline at end of file diff --git a/ratelimiter/proto/nova.ratelimit.v1.proto b/ratelimiter/proto/nova.ratelimit.v1alpha.proto similarity index 96% rename from ratelimiter/proto/nova.ratelimit.v1.proto rename to ratelimiter/proto/nova.ratelimit.v1alpha.proto index b9a0d36..5f06392 100644 --- a/ratelimiter/proto/nova.ratelimit.v1.proto +++ b/ratelimiter/proto/nova.ratelimit.v1alpha.proto @@ -1,39 +1,39 @@ -// How does this works ? -// Every request, the proxy (envoy) requests the rate-limiting service if -// the requested route bucket or global rate-limit is hit. - -syntax = "proto3"; -package nova.ratelimit.v1; - -// The reponse of a RatelimitRequest, it includes the status of the reponse and -// the bucket informations. -message RatelimitResponse { - enum Status { - OK = 0; - RATELIMITED = 1; - GLOBAL_RATELIMITED = 2; - } - Status status = 1; - bool updateAsked = 2; -} - -// Requests the ratelimit status of a route request, it also takes the -// indentifiables of the request in question. -message RatelimitRequest { - string routeName = 1; - repeated string indentifiables = 2; -} - -// Used when "updateAsked" is sed to true -// this means the bucket is unknown to the ratelimit server. -message CreateBucketData { - RatelimitRequest request = 1; - int32 limit = 2; - int32 remaining = 3; - int32 reset = 4; -} - -service RatelimitService { - rpc GetRatelimitStatus (RatelimitRequest) returns (RatelimitResponse); - rpc CreateBucket (CreateBucketData) returns (CreateBucketData); +// How does this works ? +// Every request, the proxy (envoy) requests the rate-limiting service if +// the requested route bucket or global rate-limit is hit. + +syntax = "proto3"; +package nova.ratelimit.v1; + +// The reponse of a RatelimitRequest, it includes the status of the reponse and +// the bucket informations. +message RatelimitResponse { + enum Status { + OK = 0; + RATELIMITED = 1; + GLOBAL_RATELIMITED = 2; + } + Status status = 1; + bool updateAsked = 2; +} + +// Requests the ratelimit status of a route request, it also takes the +// indentifiables of the request in question. +message RatelimitRequest { + string routeName = 1; + repeated string indentifiables = 2; +} + +// Used when "updateAsked" is sed to true +// this means the bucket is unknown to the ratelimit server. +message CreateBucketData { + RatelimitRequest request = 1; + int32 limit = 2; + int32 remaining = 3; + int32 reset = 4; +} + +service RatelimitService { + rpc GetRatelimitStatus (RatelimitRequest) returns (RatelimitResponse); + rpc CreateBucket (CreateBucketData) returns (CreateBucketData); } \ No newline at end of file diff --git a/webhook/config/default.yaml b/webhook/config/default.yaml deleted file mode 100644 index 7f88e6f..0000000 --- a/webhook/config/default.yaml +++ /dev/null @@ -1,13 +0,0 @@ -monitoring: - enabled: true - address: "0.0.0.0" - port: 5000 -nats: - host: "nova-infra-nats" -webhook: - server: - port: 8080 - address: "0.0.0.0" - discord: - client_id: 0 - public_key: "" \ No newline at end of file -- 2.39.5