From: MatthieuCoder Date: Wed, 18 Jan 2023 12:03:11 +0000 (+0400) Subject: base X-Git-Url: https://git.puffer.fish/?a=commitdiff_plain;h=7f4406d0946d409a842b4b364541380e50d92010;p=matthieu%2Fnova.git base --- diff --git a/.circleci/config.yml b/.circleci/config.yml index 6554e1f..0754c1c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,26 +1,34 @@ -# Use the latest 2.1 version of CircleCI pipeline process engine. -# See: https://circleci.com/docs/2.0/configuration-reference version: 2.1 -# Define a job to be invoked later in a workflow. -# See: https://circleci.com/docs/2.0/configuration-reference/#jobs +# Define the jobs we want to run for this project jobs: - say-hello: - # Specify the execution environment. You can specify an image from Dockerhub or use one of our Convenience Images from CircleCI's Developer Hub. - # See: https://circleci.com/docs/2.0/configuration-reference/#docker-machine-macos-windows-executor - docker: - - image: cimg/base:stable - # Add steps to the job - # See: https://circleci.com/docs/2.0/configuration-reference/#steps + build: + machine: + image: ubuntu + environment: + DOCKER_BUILDKIT: 1 + BUILDX_PLATFORMS: linux/amd64,linux/arm64,linux/ppc64le,linux/s390x,linux/386,linux/arm/v7,linux/arm/v6 steps: - checkout - run: - name: "Say hello" - command: "echo Hello, World!" + name: Unit Tests + command: make test + - run: + name: Log in to docker hub + command: | + docker login -u $DOCKER_USER -p $DOCKER_PASS + - run: + name: Build from dockerfile + command: | + TAG=edge make build + - run: + name: Push to docker hub + command: | + TAG=edge make push -# Invoke jobs via workflows -# See: https://circleci.com/docs/2.0/configuration-reference/#workflows +# Orchestrate our job run sequence workflows: - say-hello-workflow: + build_and_test: jobs: - - say-hello + - build + - test \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 3eba9a0..88525e9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,31 @@ -FROM rust AS chef -USER root -COPY .cargo .cargo -RUN cargo install cargo-chef -RUN apt-get update && apt-get install -y protobuf-compiler -WORKDIR /app - -# Planning install -FROM chef AS planner +# syntax=docker/dockerfile:1 +FROM --platform=$BUILDPLATFORM tonistiigi/xx:master AS xx +FROM --platform=$BUILDPLATFORM rust:alpine as rbuild +RUN apk add clang lld protobuf protobuf-dev git build-base mingw-w64-gcc COPY . . -RUN cargo chef prepare --recipe-path recipe.json +COPY --from=xx / / + +ARG TARGETPLATFORM +RUN xx-cargo build --release --target-dir ./build -# Building all targets -FROM chef AS builder -COPY --from=planner /app/recipe.json recipe.json +FROM --platform=$BUILDPLATFORM alpine as passwd +RUN addgroup -S nova && adduser -S nova -G nova -# Notice that we are specifying the --target flag! -RUN cargo chef cook --release --recipe-path recipe.json +FROM --platform=$BUILDPLATFORM golang:alpine as gbuild +RUN apk add clang lld +COPY --from=xx / / +ARG TARGETPLATFORM +COPY --from=rbuild /build/release/liball_in_one.a ./build/lib/liball_in_one.a COPY . . -RUN cargo build --release +RUN go build -a -ldflags '-s' -o build/bin/nova cmd/nova/nova.go -# Base os -FROM debian:latest AS runtime-base -# RUN addgroup -S nova && adduser -S nova -G nova -RUN apt-get update && apt-get install ca-certificates -y -# Final os -FROM runtime-base AS runtime +FROM scratch as component +COPY --from=passwd /etc/passwd /etc/passwd ARG COMPONENT ENV COMPONENT=${COMPONENT} -COPY --from=builder /app/target/release/${COMPONENT} /usr/local/bin/ -# USER nova +COPY --from=rbuild /build/release/${COMPONENT} /usr/local/bin/ +USER nova ENTRYPOINT /usr/local/bin/${COMPONENT} + +FROM scratch as all_in_one diff --git a/Makefile b/Makefile index 4257bd6..c1adb77 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,10 @@ ifeq ($(OS),Windows_NT) EXTENSION += .exe endif PROJECTS = $(shell find exes/ -mindepth 1 -maxdepth 1 -type d -printf '%f\n') +PLATFORMS := +ifdef BUILDX_PLATFORMS + PLATFORMS += --platform ${BUILDX_PLATFORMS} +endif # Static libraries target/release/lib%.a: libs/% @@ -27,13 +31,19 @@ build/bin/nova$(EXTENSION): build/lib/liball_in_one.a @mkdir -p build/bin go build -a -ldflags '-s' -o build/bin/nova cmd/nova/nova.go +docker-%: + + +docker: $(PROJECTS:%=docker-%) BINS=$(PROJECTS:%=build/bin/%$(EXTENSION)) -all: $(BINS) build/bin/nova$(EXTENSION) +bins: $(BINS) build/bin/nova$(EXTENSION) + +all: docker bins clean: rm -rf build rm -rf $(PROJECTS:%=target/release/%$(EXTENSION)) - rm -rf target/release/liball_in_one.a + rm -rf target/release/liball_in_one.a test: cargo test diff --git a/exes/cache/src/main.rs b/exes/cache/src/main.rs index c0dff6d..5d43a46 100644 --- a/exes/cache/src/main.rs +++ b/exes/cache/src/main.rs @@ -1,6 +1,6 @@ -use std::{error::Error, future::Future, pin::Pin}; +use std::{error::Error, future::IntoFuture}; -use async_nats::{Client, Subscriber}; +use async_nats::Subscriber; use managers::{ automoderation::Automoderation, bans::Bans, channels::Channels, guild_schedules::GuildSchedules, guilds::Guilds, integrations::Integrations, invites::Invites, @@ -42,9 +42,7 @@ struct Cache { async fn main() -> Result<(), Box> { let settings: Settings = Settings::new("cache").unwrap(); info!("loaded configuration: {:?}", settings); - let nats = - Into::> + Send>>>::into(settings.nats) - .await?; + let nats = settings.nats.into_future().await?; let mut cache = Cache::default(); diff --git a/exes/gateway/src/lib.rs b/exes/gateway/src/lib.rs index ddc4bbd..1f41e50 100644 --- a/exes/gateway/src/lib.rs +++ b/exes/gateway/src/lib.rs @@ -18,7 +18,7 @@ use shared::{ config::Settings, payloads::{CachePayload, DispatchEventTagged}, }; -use std::{convert::TryFrom, future::Future, pin::Pin, str::FromStr}; +use std::{convert::TryFrom, future::IntoFuture, str::FromStr}; use tokio::{select, sync::oneshot}; use tokio_stream::StreamExt; use tracing_opentelemetry::OpenTelemetrySpanExt; @@ -51,10 +51,7 @@ impl Component for GatewayServer { .shard(settings.shard, settings.shard_total)? .build(); - let nats = Into::> + Send>>>::into( - settings.nats, - ) - .await?; + let nats = settings.nats.into_future().await?; shard.start().await?; loop { diff --git a/exes/ratelimit/src/lib.rs b/exes/ratelimit/src/lib.rs index d1bd6e0..773d564 100644 --- a/exes/ratelimit/src/lib.rs +++ b/exes/ratelimit/src/lib.rs @@ -15,10 +15,8 @@ use config::Ratelimit; use grpc::RLServer; use leash::{AnyhowResultFuture, Component}; use proto::nova::ratelimit::ratelimiter::ratelimiter_server::RatelimiterServer; -use redis::aio::MultiplexedConnection; use shared::config::Settings; -use std::future::Future; -use std::pin::Pin; +use std::future::IntoFuture; use tokio::sync::oneshot; use tonic::transport::Server; @@ -38,10 +36,7 @@ impl Component for RatelimiterServerComponent { ) -> AnyhowResultFuture<()> { Box::pin(async move { let listening_address = settings.server.listening_adress; - let redis = Into::< - Pin> + Send>>, - >::into(settings.redis) - .await?; + let redis = settings.redis.into_future().await?; let server = RLServer::new(RedisLock::new(redis)); diff --git a/exes/webhook/src/lib.rs b/exes/webhook/src/lib.rs index 933f38e..2f0bb7c 100644 --- a/exes/webhook/src/lib.rs +++ b/exes/webhook/src/lib.rs @@ -12,13 +12,12 @@ mod config; mod handler; -use std::{future::Future, pin::Pin}; +use std::future::IntoFuture; use crate::{ config::Webhook, handler::{make_service::MakeSvc, WebhookService}, }; -use async_nats::Client; use hyper::Server; use leash::{AnyhowResultFuture, Component}; use shared::config::Settings; @@ -41,10 +40,7 @@ impl Component for WebhookServer { let bind = settings.server.listening_adress; info!("Nats connected!"); - let nats = Into::> + Send>>>::into( - settings.nats, - ) - .await?; + let nats = settings.nats.into_future().await?; let make_service = MakeSvc::new(WebhookService { config: settings.config, diff --git a/libs/shared/src/nats.rs b/libs/shared/src/nats.rs index 7d4d3d1..1d835d3 100644 --- a/libs/shared/src/nats.rs +++ b/libs/shared/src/nats.rs @@ -1,4 +1,7 @@ -use std::{future::Future, pin::Pin}; +use std::{ + future::{Future, IntoFuture}, + pin::Pin, +}; use async_nats::Client; use serde::Deserialize; @@ -8,8 +11,12 @@ pub struct Configuration { pub host: String, } -impl From for Pin> + Send>> { - fn from(value: Configuration) -> Self { - Box::pin(async move { Ok(async_nats::connect(value.host).await?) }) +impl IntoFuture for Configuration { + type Output = anyhow::Result; + + type IntoFuture = Pin + Send>>; + + fn into_future(self) -> Self::IntoFuture { + Box::pin(async move { Ok(async_nats::connect(self.host).await?) }) } } diff --git a/libs/shared/src/redis.rs b/libs/shared/src/redis.rs index 77aa97f..5cae869 100644 --- a/libs/shared/src/redis.rs +++ b/libs/shared/src/redis.rs @@ -1,18 +1,22 @@ use redis::{aio::MultiplexedConnection, Client}; use serde::Deserialize; -use std::{future::Future, pin::Pin}; +use std::{ + future::{Future, IntoFuture}, + pin::Pin, +}; #[derive(Clone, Debug, Deserialize)] pub struct Configuration { pub url: String, } -impl From - for Pin> + Send>> -{ - fn from(value: Configuration) -> Self { +impl IntoFuture for Configuration { + type Output = anyhow::Result; + type IntoFuture = Pin + Send>>; + + fn into_future(self) -> Self::IntoFuture { Box::pin(async move { - let con = Client::open(value.url)?; + let con = Client::open(self.url)?; let (multiplex, ready) = con.create_multiplexed_tokio_connection().await?; tokio::spawn(ready); diff --git a/otel/grafana/grafana.ini b/otel/grafana/grafana.ini deleted file mode 100644 index e9c5b16..0000000 --- a/otel/grafana/grafana.ini +++ /dev/null @@ -1,1170 +0,0 @@ -##################### Grafana Configuration Example ##################### -# -# Everything has defaults so you only need to uncomment things you want to -# change - -# possible values : production, development -;app_mode = production - -# instance name, defaults to HOSTNAME environment variable value or hostname if HOSTNAME var is empty -;instance_name = ${HOSTNAME} - -# force migration will run migrations that might cause dataloss -;force_migration = false - -#################################### Paths #################################### -[paths] -# Path to where grafana can store temp files, sessions, and the sqlite3 db (if that is used) -;data = /var/lib/grafana - -# Temporary files in `data` directory older than given duration will be removed -;temp_data_lifetime = 24h - -# Directory where grafana can store logs -;logs = /var/log/grafana - -# Directory where grafana will automatically scan and look for plugins -;plugins = /var/lib/grafana/plugins - -# folder that contains provisioning config files that grafana will apply on startup and while running. -provisioning = /etc/grafana/provisioning - -#################################### Server #################################### -[server] -# Protocol (http, https, h2, socket) -protocol = http - -# The ip address to bind to, empty will bind to all interfaces -;http_addr = - -# The http port to use -http_port = 3000 - -# The public facing domain name used to access grafana from a browser -domain = localhost - -# Redirect to correct domain if host header does not match domain -# Prevents DNS rebinding attacks -;enforce_domain = false - -# The full public facing url you use in browser, used for redirects and emails -# If you use reverse proxy and sub path specify full url (with sub path) -root_url = %(protocol)s://%(domain)s/grafana/ - -# Serve Grafana from subpath specified in `root_url` setting. By default it is set to `false` for compatibility reasons. -serve_from_sub_path = true - -# Log web requests -;router_logging = false - -# the path relative working path -;static_root_path = public - -# enable gzip -;enable_gzip = false - -# https certs & key file -;cert_file = -;cert_key = - -# Unix socket path -;socket = - -# CDN Url -;cdn_url = - -# Sets the maximum time using a duration format (5s/5m/5ms) before timing out read of an incoming request and closing idle connections. -# `0` means there is no timeout for reading the request. -;read_timeout = 0 - -#################################### Database #################################### -[database] -# You can configure the database connection by specifying type, host, name, user and password -# as separate properties or as on string using the url properties. - -# Either "mysql", "postgres" or "sqlite3", it's your choice -;type = sqlite3 -;host = 127.0.0.1:3306 -;name = grafana -;user = root -# If the password contains # or ; you have to wrap it with triple quotes. Ex """#password;""" -;password = - -# Use either URL or the previous fields to configure the database -# Example: mysql://user:secret@host:port/database -;url = - -# For "postgres" only, either "disable", "require" or "verify-full" -;ssl_mode = disable - -# Database drivers may support different transaction isolation levels. -# Currently, only "mysql" driver supports isolation levels. -# If the value is empty - driver's default isolation level is applied. -# For "mysql" use "READ-UNCOMMITTED", "READ-COMMITTED", "REPEATABLE-READ" or "SERIALIZABLE". -;isolation_level = - -;ca_cert_path = -;client_key_path = -;client_cert_path = -;server_cert_name = - -# For "sqlite3" only, path relative to data_path setting -;path = grafana.db - -# Max idle conn setting default is 2 -;max_idle_conn = 2 - -# Max conn setting default is 0 (mean not set) -;max_open_conn = - -# Connection Max Lifetime default is 14400 (means 14400 seconds or 4 hours) -;conn_max_lifetime = 14400 - -# Set to true to log the sql calls and execution times. -;log_queries = - -# For "sqlite3" only. cache mode setting used for connecting to the database. (private, shared) -;cache_mode = private - -# For "mysql" only if lockingMigration feature toggle is set. How many seconds to wait before failing to lock the database for the migrations, default is 0. -;locking_attempt_timeout_sec = 0 - -################################### Data sources ######################### -[datasources] -# Upper limit of data sources that Grafana will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API. -;datasource_limit = 5000 - -#################################### Cache server ############################# -[remote_cache] -# Either "redis", "memcached" or "database" default is "database" -;type = database - -# cache connectionstring options -# database: will use Grafana primary database. -# redis: config like redis server e.g. `addr=127.0.0.1:6379,pool_size=100,db=0,ssl=false`. Only addr is required. ssl may be 'true', 'false', or 'insecure'. -# memcache: 127.0.0.1:11211 -;connstr = - -#################################### Data proxy ########################### -[dataproxy] - -# This enables data proxy logging, default is false -;logging = false - -# How long the data proxy waits to read the headers of the response before timing out, default is 30 seconds. -# This setting also applies to core backend HTTP data sources where query requests use an HTTP client with timeout set. -;timeout = 30 - -# How long the data proxy waits to establish a TCP connection before timing out, default is 10 seconds. -;dialTimeout = 10 - -# How many seconds the data proxy waits before sending a keepalive probe request. -;keep_alive_seconds = 30 - -# How many seconds the data proxy waits for a successful TLS Handshake before timing out. -;tls_handshake_timeout_seconds = 10 - -# How many seconds the data proxy will wait for a server's first response headers after -# fully writing the request headers if the request has an "Expect: 100-continue" -# header. A value of 0 will result in the body being sent immediately, without -# waiting for the server to approve. -;expect_continue_timeout_seconds = 1 - -# Optionally limits the total number of connections per host, including connections in the dialing, -# active, and idle states. On limit violation, dials will block. -# A value of zero (0) means no limit. -;max_conns_per_host = 0 - -# The maximum number of idle connections that Grafana will keep alive. -;max_idle_connections = 100 - -# How many seconds the data proxy keeps an idle connection open before timing out. -;idle_conn_timeout_seconds = 90 - -# If enabled and user is not anonymous, data proxy will add X-Grafana-User header with username into the request, default is false. -;send_user_header = false - -# Limit the amount of bytes that will be read/accepted from responses of outgoing HTTP requests. -;response_limit = 0 - -# Limits the number of rows that Grafana will process from SQL data sources. -;row_limit = 1000000 - -#################################### Analytics #################################### -[analytics] -# Server reporting, sends usage counters to stats.grafana.org every 24 hours. -# No ip addresses are being tracked, only simple counters to track -# running instances, dashboard and error counts. It is very helpful to us. -# Change this option to false to disable reporting. -;reporting_enabled = true - -# The name of the distributor of the Grafana instance. Ex hosted-grafana, grafana-labs -;reporting_distributor = grafana-labs - -# Set to false to disable all checks to https://grafana.com -# for new versions of grafana. The check is used -# in some UI views to notify that a grafana update exists. -# This option does not cause any auto updates, nor send any information -# only a GET request to https://raw.githubusercontent.com/grafana/grafana/main/latest.json to get the latest version. -;check_for_updates = true - -# Set to false to disable all checks to https://grafana.com -# for new versions of plugins. The check is used -# in some UI views to notify that a plugin update exists. -# This option does not cause any auto updates, nor send any information -# only a GET request to https://grafana.com to get the latest versions. -;check_for_plugin_updates = true - -# Google Analytics universal tracking code, only enabled if you specify an id here -;google_analytics_ua_id = - -# Google Tag Manager ID, only enabled if you specify an id here -;google_tag_manager_id = - -# Rudderstack write key, enabled only if rudderstack_data_plane_url is also set -;rudderstack_write_key = - -# Rudderstack data plane url, enabled only if rudderstack_write_key is also set -;rudderstack_data_plane_url = - -# Rudderstack SDK url, optional, only valid if rudderstack_write_key and rudderstack_data_plane_url is also set -;rudderstack_sdk_url = - -# Rudderstack Config url, optional, used by Rudderstack SDK to fetch source config -;rudderstack_config_url = - -# Controls if the UI contains any links to user feedback forms -;feedback_links_enabled = true - -#################################### Security #################################### -[security] -# disable creation of admin user on first start of grafana -;disable_initial_admin_creation = false - -# default admin user, created on startup -;admin_user = admin - -# default admin password, can be changed before first start of grafana, or in profile settings -;admin_password = admin - -# used for signing -;secret_key = SW2YcwTIb9zpOOhoPsMm - -# current key provider used for envelope encryption, default to static value specified by secret_key -;encryption_provider = secretKey.v1 - -# list of configured key providers, space separated (Enterprise only): e.g., awskms.v1 azurekv.v1 -;available_encryption_providers = - -# disable gravatar profile images -;disable_gravatar = false - -# data source proxy whitelist (ip_or_domain:port separated by spaces) -;data_source_proxy_whitelist = - -# disable protection against brute force login attempts -;disable_brute_force_login_protection = false - -# set to true if you host Grafana behind HTTPS. default is false. -;cookie_secure = false - -# set cookie SameSite attribute. defaults to `lax`. can be set to "lax", "strict", "none" and "disabled" -;cookie_samesite = lax - -# set to true if you want to allow browsers to render Grafana in a ,