version: 2.1
+orbs:
+ win: circleci/windows@2.2.0
+
commands:
- setup-bazel:
+ setup-bazel-linux:
description: |
Setup the Bazel build system used for building Android projects
steps:
- run:
name: Install Bazel from Apt
command: sudo apt update && sudo apt install bazel
+ setup-ci-linux:
+ description: |
+ Prepare all the requirements for the CI
+ steps:
+ - checkout
+ - setup-bazel
+ - setup_remote_docker:
+ version: 19.03.13
+ docker_layer_caching: true
+ - run:
+ name: Docker login
+ command: echo "$GHCR_PASS" | docker login ghcr.io -u "$GHCR_USER" --password-stdin
+ build-and-test:
+ description: |
+ Builds and test
+ steps:
+ - run:
+ name: "Test"
+ command: "bazel test //:tests"
+ - run:
+ name: "Build"
+ command: "bazel build //:packages"
+ - run:
+ name: "Move artifacts"
+ command: |
+ mkdir ~/project/artifacts
+ mv ~/project/bazel-bin/packages* ~/project/artifacts
+ - store_artifacts:
+ path: ~/project/artifacts
jobs:
- build:
+ build-linux-arm64:
machine:
- image: 'ubuntu-2004:202010-01'
+ image: ubuntu-2004:202101-01
+ resource_class: arm.medium
steps:
- - checkout
- - setup-bazel
+ - setup-ci-linux
- restore_cache:
keys:
- - bazel-cache-{{ .Branch }}
+ - bazel-cache-linux-arm-{{ .Branch }}
+ - build-and-test
- run:
- name: "Build"
- command: "bazel build //:packages"
+ name: Publish docker images
+ command: |
+ bazel run --define docker_repo=ghcr.io --define docker_tag=arm-{{ .Branch }} //:container_publish
+
+ build-linux-x86:
+ machine:
+ image: ubuntu-2004:202010-01
+ steps:
+ - setup-ci-linux
- save_cache:
paths:
- ~/.cache/bazel
- key: bazel-cache-{{ .Branch }}
+ key: bazel-cache-linux-x86-{{ .Branch }}
+ - build-and-test
+ - run:
+ name: Publish docker images
+ command: |
+ bazel run --define docker_repo=ghcr.io --define docker_tag={{ .Branch }} //:container_publish
+ build-windows:
+ executor:
+ name: win/default
+ shell: powershell.exe
+ steps:
+ - checkout
+ - run: systeminfo
+ - run: choco install bazel
+ - restore_cache:
+ keys:
+ - bazel-cache-windows-{{ .Branch }}
+ - build-and-test
+ build-macos:
+ macos:
+ xcode: 11.3.0
+ steps:
+ - checkout
+ - run: brew install bazel
+ - restore_cache:
+ keys:
+ - bazel-cache-macos-{{ .Branch }}
+ - run:
+ name: "Build"
+ command: "bazel build //:packages"
- run:
name: "Move artifacts"
command: |
workflows:
build-workflow:
jobs:
- - build
+ - build-x86
+ - build-arm64
+ - build-windows
+ - build-macos
ARG NONROOT_USER=vscode
# Install required programs for the container
-RUN apt update -y && apt install apt-transport-https curl sudo gnupg python build-essential ca-certificates lsb-release -y && \
+RUN apt update -y && apt install libssl-dev pkg-config apt-transport-https curl sudo gnupg python build-essential ca-certificates lsb-release -y && \
# Add bazel repository gpg keys
curl -fsSL https://bazel.build/bazel-release.pub.gpg | gpg --dearmor -o /etc/apt/trusted.gpg.d/bazel.gpg && \
# Add docker repository gpg keys
"memchr",
]
+[[package]]
+name = "arc-swap"
+version = "1.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6df5aef5c5830360ce5218cecb8f018af3438af5686ae945094affc86fdec63"
+
[[package]]
name = "arrayvec"
version = "0.5.2"
checksum = "a909e4d93292cd8e9c42e189f61681eff9d67b6541f96b8a1a737f23737bd001"
dependencies = [
"bytes",
+ "futures-core",
"memchr",
+ "pin-project-lite",
+ "tokio",
+ "tokio-util",
]
[[package]]
"nats",
"pretty_env_logger",
"prometheus",
+ "redis",
"serde 1.0.130",
"testcontainers",
"tokio",
"libc",
]
+[[package]]
+name = "crc16"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff"
+
[[package]]
name = "crossbeam-channel"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "202c5bf92cad3d57605c366e644a7fbf305a83f19754fc66678c6265dcc9b8b4"
dependencies = [
+ "arc-swap",
"async-trait",
+ "bytes",
"combine",
+ "crc16",
"dtoa",
+ "futures",
+ "futures-util",
"itoa",
"percent-encoding",
+ "pin-project-lite",
+ "rand 0.8.4",
"sha1",
+ "tokio",
+ "tokio-util",
"url",
]
"futures-util",
"hyper",
"hyper-tls",
+ "lazy_static",
"serde 1.0.130",
"tokio",
+ "xxhash-rust",
]
[[package]]
"common",
"hex",
"hyper",
+ "lazy_static",
"libc",
"libsodium-sys",
"serde 1.0.130",
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
+[[package]]
+name = "xxhash-rust"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e575e15bedf6e57b5c2d763ffc6c3c760143466cbd09d762d539680ab5992ded"
+
[[package]]
name = "yaml-rust"
version = "0.4.5"
"webhook": {
"hex": "@raze__hex__0_4_3//:hex",
"hyper": "@raze__hyper__0_14_12//:hyper",
+ "lazy_static": "@raze__lazy_static__1_4_0//:lazy_static",
"libc": "@raze__libc__0_2_101//:libc",
"libsodium-sys": "@raze__libsodium_sys__0_2_7//:libsodium_sys",
"serde": "@raze__serde__1_0_130//:serde",
"nats": "@raze__nats__0_15_2//:nats",
"pretty_env_logger": "@raze__pretty_env_logger__0_4_0//:pretty_env_logger",
"prometheus": "@raze__prometheus__0_12_0//:prometheus",
+ "redis": "@raze__redis__0_21_2//:redis",
"serde": "@raze__serde__1_0_130//:serde",
"testcontainers": "@raze__testcontainers__0_12_0//:testcontainers",
"tokio": "@raze__tokio__1_11_0//:tokio",
"futures-util": "@raze__futures_util__0_3_17//:futures_util",
"hyper": "@raze__hyper__0_14_12//:hyper",
"hyper-tls": "@raze__hyper_tls__0_5_0//:hyper_tls",
+ "lazy_static": "@raze__lazy_static__1_4_0//:lazy_static",
"serde": "@raze__serde__1_0_130//:serde",
"tokio": "@raze__tokio__1_11_0//:tokio",
+ "xxhash-rust": "@raze__xxhash_rust__0_8_2//:xxhash_rust",
},
"": {
"libc": "@raze__libc__0_2_101//:libc",
build_file = Label("//cargo/remote:BUILD.aho-corasick-0.7.18.bazel"),
)
+ maybe(
+ http_archive,
+ name = "raze__arc_swap__1_4_0",
+ url = "https://crates.io/api/v1/crates/arc-swap/1.4.0/download",
+ type = "tar.gz",
+ sha256 = "e6df5aef5c5830360ce5218cecb8f018af3438af5686ae945094affc86fdec63",
+ strip_prefix = "arc-swap-1.4.0",
+ build_file = Label("//cargo/remote:BUILD.arc-swap-1.4.0.bazel"),
+ )
+
maybe(
http_archive,
name = "raze__arrayvec__0_5_2",
build_file = Label("//cargo/remote:BUILD.cpufeatures-0.2.1.bazel"),
)
+ maybe(
+ http_archive,
+ name = "raze__crc16__0_4_0",
+ url = "https://crates.io/api/v1/crates/crc16/0.4.0/download",
+ type = "tar.gz",
+ sha256 = "338089f42c427b86394a5ee60ff321da23a5c89c9d89514c829687b26359fcff",
+ strip_prefix = "crc16-0.4.0",
+ build_file = Label("//cargo/remote:BUILD.crc16-0.4.0.bazel"),
+ )
+
maybe(
http_archive,
name = "raze__crossbeam_channel__0_5_1",
build_file = Label("//cargo/remote:BUILD.winapi-x86_64-pc-windows-gnu-0.4.0.bazel"),
)
+ maybe(
+ http_archive,
+ name = "raze__xxhash_rust__0_8_2",
+ url = "https://crates.io/api/v1/crates/xxhash-rust/0.8.2/download",
+ type = "tar.gz",
+ sha256 = "e575e15bedf6e57b5c2d763ffc6c3c760143466cbd09d762d539680ab5992ded",
+ strip_prefix = "xxhash-rust-0.8.2",
+ build_file = Label("//cargo/remote:BUILD.xxhash-rust-0.8.2.bazel"),
+ )
+
maybe(
http_archive,
name = "raze__yaml_rust__0_4_5",
--- /dev/null
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+# buildifier: disable=load
+load(
+ "@rules_rust//rust:rust.bzl",
+ "rust_binary",
+ "rust_library",
+ "rust_test",
+)
+
+package(default_visibility = [
+ # Public for visibility by "@raze__crate__version//" targets.
+ #
+ # Prefer access through "//cargo", which limits external
+ # visibility to explicit Cargo.toml dependencies.
+ "//visibility:public",
+])
+
+licenses([
+ "notice", # Apache-2.0 from expression "Apache-2.0 OR MIT"
+])
+
+# Generated Targets
+
+# Unsupported target "background" with type "bench" omitted
+
+# Unsupported target "int-access" with type "bench" omitted
+
+# Unsupported target "track" with type "bench" omitted
+
+rust_library(
+ name = "arc_swap",
+ srcs = glob(["**/*.rs"]),
+ crate_features = [
+ ],
+ crate_root = "src/lib.rs",
+ crate_type = "lib",
+ data = [],
+ edition = "2018",
+ rustc_flags = [
+ "--cap-lints=allow",
+ ],
+ tags = [
+ "cargo-raze",
+ "manual",
+ ],
+ version = "1.4.0",
+ # buildifier: leave-alone
+ deps = [
+ ],
+)
+
+# Unsupported target "random" with type "test" omitted
+
+# Unsupported target "stress" with type "test" omitted
rust_library(
name = "combine",
srcs = glob(["**/*.rs"]),
+ aliases = {
+ "@raze__futures_core__0_3_17//:futures_core": "futures_core_03",
+ "@raze__tokio__1_11_0//:tokio": "tokio_dep",
+ },
crate_features = [
"alloc",
"bytes",
+ "futures-core-03",
+ "pin-project-lite",
"std",
+ "tokio",
+ "tokio-dep",
+ "tokio-util",
],
crate_root = "src/lib.rs",
crate_type = "lib",
# buildifier: leave-alone
deps = [
"@raze__bytes__1_1_0//:bytes",
+ "@raze__futures_core__0_3_17//:futures_core",
"@raze__memchr__2_4_1//:memchr",
+ "@raze__pin_project_lite__0_2_7//:pin_project_lite",
+ "@raze__tokio__1_11_0//:tokio",
+ "@raze__tokio_util__0_6_8//:tokio_util",
],
)
--- /dev/null
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+# buildifier: disable=load
+load(
+ "@rules_rust//rust:rust.bzl",
+ "rust_binary",
+ "rust_library",
+ "rust_test",
+)
+
+package(default_visibility = [
+ # Public for visibility by "@raze__crate__version//" targets.
+ #
+ # Prefer access through "//cargo", which limits external
+ # visibility to explicit Cargo.toml dependencies.
+ "//visibility:public",
+])
+
+licenses([
+ "notice", # MIT from expression "MIT"
+])
+
+# Generated Targets
+# buildifier: disable=out-of-order-load
+# buildifier: disable=load-on-top
+load(
+ "@rules_rust//cargo:cargo_build_script.bzl",
+ "cargo_build_script",
+)
+
+cargo_build_script(
+ name = "crc16_build_script",
+ srcs = glob(["**/*.rs"]),
+ build_script_env = {
+ },
+ crate_features = [
+ ],
+ crate_root = "build.rs",
+ data = glob(["**"]),
+ edition = "2015",
+ rustc_flags = [
+ "--cap-lints=allow",
+ ],
+ tags = [
+ "cargo-raze",
+ "manual",
+ ],
+ version = "0.4.0",
+ visibility = ["//visibility:private"],
+ deps = [
+ ],
+)
+
+rust_library(
+ name = "crc16",
+ srcs = glob(["**/*.rs"]),
+ crate_features = [
+ ],
+ crate_root = "src/lib.rs",
+ crate_type = "lib",
+ data = [],
+ edition = "2015",
+ rustc_flags = [
+ "--cap-lints=allow",
+ ],
+ tags = [
+ "cargo-raze",
+ "manual",
+ ],
+ version = "0.4.0",
+ # buildifier: leave-alone
+ deps = [
+ ":crc16_build_script",
+ ],
+)
srcs = glob(["**/*.rs"]),
crate_features = [
"acl",
+ "aio",
+ "arc-swap",
+ "bytes",
+ "cluster",
+ "connection-manager",
+ "crc16",
"default",
+ "futures",
+ "futures-util",
"geospatial",
+ "pin-project-lite",
+ "rand",
"script",
"sha1",
"streams",
+ "tokio",
+ "tokio-comp",
+ "tokio-util",
],
crate_root = "src/lib.rs",
crate_type = "lib",
version = "0.21.2",
# buildifier: leave-alone
deps = [
+ "@raze__arc_swap__1_4_0//:arc_swap",
+ "@raze__bytes__1_1_0//:bytes",
"@raze__combine__4_6_1//:combine",
+ "@raze__crc16__0_4_0//:crc16",
"@raze__dtoa__0_4_8//:dtoa",
+ "@raze__futures__0_3_17//:futures",
+ "@raze__futures_util__0_3_17//:futures_util",
"@raze__itoa__0_4_8//:itoa",
"@raze__percent_encoding__2_1_0//:percent_encoding",
+ "@raze__pin_project_lite__0_2_7//:pin_project_lite",
+ "@raze__rand__0_8_4//:rand",
"@raze__sha1__0_6_0//:sha1",
+ "@raze__tokio__1_11_0//:tokio",
+ "@raze__tokio_util__0_6_8//:tokio_util",
"@raze__url__2_2_2//:url",
],
)
crate_features = [
"codec",
"default",
+ "io",
],
crate_root = "src/lib.rs",
crate_type = "lib",
--- /dev/null
+"""
+@generated
+cargo-raze crate build file.
+
+DO NOT EDIT! Replaced on runs of cargo-raze
+"""
+
+# buildifier: disable=load
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
+# buildifier: disable=load
+load(
+ "@rules_rust//rust:rust.bzl",
+ "rust_binary",
+ "rust_library",
+ "rust_test",
+)
+
+package(default_visibility = [
+ # Public for visibility by "@raze__crate__version//" targets.
+ #
+ # Prefer access through "//cargo", which limits external
+ # visibility to explicit Cargo.toml dependencies.
+ "//visibility:public",
+])
+
+licenses([
+ "notice", # BSL-1.0 from expression "BSL-1.0"
+])
+
+# Generated Targets
+
+# Unsupported target "compare_c" with type "bench" omitted
+
+rust_library(
+ name = "xxhash_rust",
+ srcs = glob(["**/*.rs"]),
+ crate_features = [
+ "xxh32",
+ ],
+ crate_root = "src/lib.rs",
+ crate_type = "lib",
+ data = [],
+ edition = "2018",
+ rustc_flags = [
+ "--cap-lints=allow",
+ ],
+ tags = [
+ "cargo-raze",
+ "manual",
+ ],
+ version = "0.8.2",
+ # buildifier: leave-alone
+ deps = [
+ ],
+)
+
+# Unsupported target "assert_correctness" with type "test" omitted
tokio = { version = "1", features = ["full"] }
prometheus = "0.12.0"
nats = "0.15.2"
-testcontainers = "0.12.0"
\ No newline at end of file
+testcontainers = "0.12.0"
+
+[dependencies.redis]
+version = "*"
+features = ["cluster", "connection-manager", "tokio-comp"]
],
)
+alias(
+ name = "redis",
+ actual = "@raze__redis__0_21_2//:redis",
+ tags = [
+ "cargo-raze",
+ "manual",
+ ],
+)
+
alias(
name = "serde",
actual = "@raze__serde__1_0_130//:serde",
pub config: T,
pub monitoring: crate::monitoring::MonitoringConfiguration,
pub nats: crate::nats::NatsConfiguration,
+ pub redis: crate::redis::RedisConfiguration,
}
///
pub mod nats;
pub mod payloads;
pub mod error;
+pub mod redis;
pub use log as log;
pub use serde as serde;
pub use ::config as config_crate;
pub use prometheus as prometheus;
pub use ::nats as nats_crate;
-pub use testcontainers as testcontainers;
\ No newline at end of file
+pub use testcontainers as testcontainers;
+pub use ::redis as redis_crate;
\ No newline at end of file
--- /dev/null
+use redis::Client;
+use serde::Deserialize;
+
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct RedisConfiguration {
+ pub url: String,
+}
+
+// Allows the configuration to directly create a nats connection
+impl Into<Client> for RedisConfiguration {
+ fn into(self) -> Client {
+ redis::Client::open(self.url).unwrap()
+ }
+}
serde = { version = "1.0.8", features = ["derive"] }
futures-util = "0.3.17"
hyper-tls = "0.5.0"
+lazy_static = "1.4.0"
+xxhash-rust = { version = "0.8.2", features = ["xxh32"] }
\ No newline at end of file
],
)
+alias(
+ name = "lazy_static",
+ actual = "@raze__lazy_static__1_4_0//:lazy_static",
+ tags = [
+ "cargo-raze",
+ "manual",
+ ],
+)
+
alias(
name = "serde",
actual = "@raze__serde__1_0_130//:serde",
"manual",
],
)
+
+alias(
+ name = "xxhash_rust",
+ actual = "@raze__xxhash_rust__0_8_2//:xxhash_rust",
+ tags = [
+ "cargo-raze",
+ "manual",
+ ],
+)
use std::{convert::Infallible, sync::Arc};
-use crate::config::Config;
-use common::{config::Settings, log::{error, info}};
+use crate::{config::Config, ratelimit::Ratelimiter};
+use common::{
+ config::Settings,
+ log::{error, info},
+ redis_crate::Client,
+};
use hyper::{server::conn::AddrStream, service::make_service_fn, Server};
use std::net::ToSocketAddrs;
+use tokio::sync::Mutex;
use crate::proxy::ServiceProxy;
mod config;
mod proxy;
+mod ratelimit;
#[tokio::main]
async fn main() {
let settings: Settings<Config> = Settings::new("rest").unwrap();
let config = Arc::new(settings.config);
+ let redis_client: Client = settings.redis.into();
+ let redis = Arc::new(Mutex::new(
+ redis_client.get_async_connection().await.unwrap(),
+ ));
+ let ratelimiter = Arc::new(Ratelimiter::new(redis));
let addr = format!("{}:{}", config.server.address, config.server.port)
.to_socket_addrs()
.unwrap();
let service_fn = make_service_fn(move |_: &AddrStream| {
- let service_proxy = ServiceProxy::new(config.clone());
+ let service_proxy = ServiceProxy::new(config.clone(), ratelimiter.clone());
async move { Ok::<_, Infallible>(service_proxy) }
});
-use crate::config::Config;
+use crate::{config::Config, ratelimit::Ratelimiter};
use futures_util::future::TryFutureExt;
use hyper::{
client::HttpConnector, header::HeaderValue, http::uri::Parts, service::Service, Body, Client,
#[derive(Clone)]
pub struct ServiceProxy {
client: Client<HttpsConnector<HttpConnector>>,
+ ratelimiter: Arc<Ratelimiter>,
config: Arc<Config>,
}
}
impl ServiceProxy {
- pub fn new(config: Arc<Config>) -> Self {
+ pub fn new(config: Arc<Config>, ratelimiter: Arc<Ratelimiter>) -> Self {
let https = HttpsConnector::new();
let client = Client::builder().build::<_, hyper::Body>(https);
- ServiceProxy { client, config }
+ ServiceProxy { client, config, ratelimiter }
}
}
--- /dev/null
+use common::redis_crate::{AsyncCommands, RedisError, aio::Connection};
+use hyper::{Body, Request};
+use tokio::sync::Mutex;
+use std::sync::Arc;
+use xxhash_rust::xxh32::xxh32;
+
+pub struct Ratelimiter {
+ redis: Arc<Mutex<Connection>>
+}
+
+impl Ratelimiter {
+ pub fn new(redis: Arc<Mutex<Connection>>) -> Ratelimiter {
+ return Ratelimiter {
+ redis
+ }
+ }
+
+ pub async fn check(&mut self,request: Request<Body>) -> bool {
+ // we lookup if the route hash is stored in the redis table
+ let path = request.uri().path();
+ let hash = xxh32(path.as_bytes(), 32);
+ let key = format!("nova:rest:ratelimit:url_store:{}", hash);
+ let mut redis = self.redis.lock().await;
+ let value: Result<String, RedisError> = redis.get(key).await;
+
+ match value {
+ Ok(_) => true,
+ Err(error) => false,
+ }
+ }
+}
projects / matthieu / nova.git / commitdiff