Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 26 additions & 0 deletions .github/workflow-template/template.yml
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,12 @@ jobs:
name: meta-node
path: ./.risingwave/bin

- uses: actions/download-artifact@v2
name: Download rust frontend binary
with:
name: frontend-v2
path: ./.risingwave/bin
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm thinking of uploading all ***-node into one artifact zip...

Another idea is to create an all-in-one binary for all our nodes. IIRC, every executable we created is ~200MB, but most debug symbols are the same...

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

LGTM


- uses: actions/download-artifact@v2
name: Download frontend binary
with:
Expand All @@ -93,6 +99,7 @@ jobs:
run: |
chmod +x ./.risingwave/bin/meta-node
chmod +x ./.risingwave/bin/compute-node
chmod +x ./.risingwave/bin/frontend-v2
chmod +x ./rust/target/debug/risedev-playground

- name: Install required components
Expand Down Expand Up @@ -138,6 +145,18 @@ jobs:
~/cargo-make/makers logs
~/cargo-make/makers check-logs

- name: e2e test w/ rust frontend
run: |
~/cargo-make/makers clean-data
~/cargo-make/makers ci-v2
timeout 3m sqllogictest -p 4566 -d dev './e2e_test/v2/*.slt'

- name: Kill cluster
run: |
~/cargo-make/makers k
~/cargo-make/makers logs
~/cargo-make/makers check-logs

- name: e2e test w/ Hummock w/o cache 3-node
run: |
~/cargo-make/makers clean-data
Expand Down Expand Up @@ -449,6 +468,13 @@ jobs:
path: |
rust/target/debug/meta-node

- uses: actions/upload-artifact@v2
with:
name: frontend-v2
if-no-files-found: error
path: |
rust/target/debug/frontend-v2

- uses: actions/upload-artifact@v2
with:
name: risingwave_regress_test
Expand Down
22 changes: 22 additions & 0 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,11 @@ jobs:
with:
name: meta-node
path: ./.risingwave/bin
- uses: actions/download-artifact@v2
name: Download rust frontend binary
with:
name: frontend-v2
path: ./.risingwave/bin
- uses: actions/download-artifact@v2
name: Download frontend binary
with:
Expand All @@ -87,6 +92,7 @@ jobs:
run: |
chmod +x ./.risingwave/bin/meta-node
chmod +x ./.risingwave/bin/compute-node
chmod +x ./.risingwave/bin/frontend-v2
chmod +x ./rust/target/debug/risedev-playground
- name: Install required components
run: |
Expand Down Expand Up @@ -118,6 +124,16 @@ jobs:
~/cargo-make/makers k
~/cargo-make/makers logs
~/cargo-make/makers check-logs
- name: e2e test w/ rust frontend
run: |
~/cargo-make/makers clean-data
~/cargo-make/makers ci-v2
timeout 3m sqllogictest -p 4566 -d dev './e2e_test/v2/*.slt'
- name: Kill cluster
run: |
~/cargo-make/makers k
~/cargo-make/makers logs
~/cargo-make/makers check-logs
- name: e2e test w/ Hummock w/o cache 3-node
run: |
~/cargo-make/makers clean-data
Expand Down Expand Up @@ -376,6 +392,12 @@ jobs:
if-no-files-found: error
path: |
rust/target/debug/meta-node
- uses: actions/upload-artifact@v2
with:
name: frontend-v2
if-no-files-found: error
path: |
rust/target/debug/frontend-v2
- uses: actions/upload-artifact@v2
with:
name: risingwave_regress_test
Expand Down
22 changes: 22 additions & 0 deletions .github/workflows/pull-request.yml
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,11 @@ jobs:
with:
name: meta-node
path: ./.risingwave/bin
- uses: actions/download-artifact@v2
name: Download rust frontend binary
with:
name: frontend-v2
path: ./.risingwave/bin
- uses: actions/download-artifact@v2
name: Download frontend binary
with:
Expand All @@ -86,6 +91,7 @@ jobs:
run: |
chmod +x ./.risingwave/bin/meta-node
chmod +x ./.risingwave/bin/compute-node
chmod +x ./.risingwave/bin/frontend-v2
chmod +x ./rust/target/debug/risedev-playground
- name: Install required components
run: |
Expand Down Expand Up @@ -117,6 +123,16 @@ jobs:
~/cargo-make/makers k
~/cargo-make/makers logs
~/cargo-make/makers check-logs
- name: e2e test w/ rust frontend
run: |
~/cargo-make/makers clean-data
~/cargo-make/makers ci-v2
timeout 3m sqllogictest -p 4566 -d dev './e2e_test/v2/*.slt'
- name: Kill cluster
run: |
~/cargo-make/makers k
~/cargo-make/makers logs
~/cargo-make/makers check-logs
- name: e2e test w/ Hummock w/o cache 3-node
run: |
~/cargo-make/makers clean-data
Expand Down Expand Up @@ -375,6 +391,12 @@ jobs:
if-no-files-found: error
path: |
rust/target/debug/meta-node
- uses: actions/upload-artifact@v2
with:
name: frontend-v2
if-no-files-found: error
path: |
rust/target/debug/frontend-v2
- uses: actions/upload-artifact@v2
with:
name: risingwave_regress_test
Expand Down
11 changes: 11 additions & 0 deletions Makefile.toml
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,10 @@ script = '''
set -e
rm -f "${PREFIX_BIN}/compute-node"
rm -f "${PREFIX_BIN}/meta-node"
rm -f "${PREFIX_BIN}/frontend-v2"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/compute-node" "${PREFIX_BIN}/compute-node"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/meta-node" "${PREFIX_BIN}/meta-node"
ln -s "$(pwd)/rust/target/${BUILD_MODE_DIR}/frontend-v2" "${PREFIX_BIN}/frontend-v2"
'''

[tasks.build-risingwave]
Expand Down Expand Up @@ -159,6 +161,15 @@ alias = "dev"
[tasks.d]
alias = "dev"

[tasks.ci-v2]
category = "RiseDev - Start"
dependencies = [
"pre-start-playground"
]
command = "rust/target/${BUILD_MODE_DIR}/risedev-playground"
args = ["ci-v2"]
description = "Start a RisingWave playground ci-v2 config"

[tasks.dev]
category = "RiseDev - Start"
dependencies = [
Expand Down
5 changes: 5 additions & 0 deletions e2e_test/v2/basic.slt
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
statement ok
create table t (v1 int not null);

statement ok
drop table t;
23 changes: 23 additions & 0 deletions risedev.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,11 @@
risedev:

ci-v2:
- use: minio
- use: meta-node
- use: compute-node
- use: frontend-v2

# The CI-3node configuration will start 3 compute nodes, 1 meta node, 1 frontend and 1 MinIO.
ci-3node:
- use: minio
Expand Down Expand Up @@ -167,6 +174,22 @@ template:

# If `user-managed` is true, this service will be started by user with the above config
user-managed: false

frontend-v2:
# Listen address of frontend
address: "127.0.0.1"

# Listen port of frontend
port: 4566

# Id of this instance
id: frontend-v2

# Meta-nodes used by this frontend instance
provide-meta-node: "meta-node*"

# If `user-managed` is true, this service will be started by user with the above config
user-managed: false

grafana:
# Listen address of Grafana
Expand Down
5 changes: 5 additions & 0 deletions rust/frontend/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,8 @@ uuid = "0.8"
[dev-dependencies]
serial_test = "*"
tempfile = "3"

[[bin]]
name = "frontend-v2"
path = "src/bin/frontend_node.rs"

23 changes: 21 additions & 2 deletions rust/risedev/src/bin/risedev-playground.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@ use indicatif::{MultiProgress, ProgressBar};
use risedev::util::complete_spin;
use risedev::{
ComputeNodeService, ConfigExpander, ConfigureTmuxTask, EnsureStopService, ExecuteContext,
FrontendService, GrafanaService, JaegerService, MetaNodeService, MinioService,
PrometheusService, ServiceConfig, Task, RISEDEV_SESSION_NAME,
FrontendService, FrontendServiceV2, GrafanaService, JaegerService, MetaNodeService,
MinioService, PrometheusService, ServiceConfig, Task, RISEDEV_SESSION_NAME,
};
use tempfile::tempdir;
use yaml_rust::YamlEmitter;
Expand Down Expand Up @@ -103,6 +103,7 @@ fn task_main(
ServiceConfig::ComputeNode(c) => (c.port, c.id.clone()),
ServiceConfig::MetaNode(c) => (c.port, c.id.clone()),
ServiceConfig::Frontend(c) => (c.port, c.id.clone()),
ServiceConfig::FrontendV2(c) => (c.port, c.id.clone()),
ServiceConfig::Grafana(c) => (c.port, c.id.clone()),
ServiceConfig::Jaeger(c) => (c.dashboard_port, c.id.clone()),
});
Expand Down Expand Up @@ -183,6 +184,24 @@ fn task_main(
.bold()
)?;
}
ServiceConfig::FrontendV2(c) => {
let mut ctx =
ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
let mut service = FrontendServiceV2::new(c.clone())?;
service.execute(&mut ctx)?;
let mut task = risedev::ConfigureGrpcNodeTask::new(c.port, c.user_managed)?;
task.execute(&mut ctx)?;
ctx.pb
.set_message(format!("api postgres://{}:{}/", c.address, c.port));

writeln!(
log_buffer,
"* Run {} to start Postgres interactive shell.",
style(format!("psql -h localhost -p {} -d dev", c.port))
.blue()
.bold()
)?;
}
ServiceConfig::Grafana(c) => {
let mut ctx =
ExecuteContext::new(&mut logger, manager.new_progress(), status_dir.clone());
Expand Down
1 change: 1 addition & 0 deletions rust/risedev/src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,7 @@ impl ConfigExpander {
let result = match use_type.as_str() {
"minio" => ServiceConfig::Minio(serde_yaml::from_str(&out_str)?),
"frontend" => ServiceConfig::Frontend(serde_yaml::from_str(&out_str)?),
"frontend-v2" => ServiceConfig::FrontendV2(serde_yaml::from_str(&out_str)?),
"compute-node" => ServiceConfig::ComputeNode(serde_yaml::from_str(&out_str)?),
"meta-node" => ServiceConfig::MetaNode(serde_yaml::from_str(&out_str)?),
"prometheus" => ServiceConfig::Prometheus(serde_yaml::from_str(&out_str)?),
Expand Down
2 changes: 2 additions & 0 deletions rust/risedev/src/config/service_config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ pub enum ServiceConfig {
ComputeNode(ComputeNodeConfig),
MetaNode(MetaNodeConfig),
Frontend(FrontendConfig),
FrontendV2(FrontendConfig),
Minio(MinioConfig),
Prometheus(PrometheusConfig),
Grafana(GrafanaConfig),
Expand All @@ -96,6 +97,7 @@ impl ServiceConfig {
Self::ComputeNode(c) => &c.id,
Self::MetaNode(c) => &c.id,
Self::Frontend(c) => &c.id,
Self::FrontendV2(c) => &c.id,
Self::Minio(c) => &c.id,
Self::Prometheus(c) => &c.id,
Self::Grafana(c) => &c.id,
Expand Down
2 changes: 2 additions & 0 deletions rust/risedev/src/task.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ mod compute_node_service;
mod configure_tmux_service;
mod ensure_stop_service;
mod frontend_service;
mod frontend_service_v2;
mod grafana_service;
mod jaeger_service;
mod meta_node_service;
Expand All @@ -24,6 +25,7 @@ pub use self::compute_node_service::*;
pub use self::configure_tmux_service::*;
pub use self::ensure_stop_service::*;
pub use self::frontend_service::*;
pub use self::frontend_service_v2::*;
pub use self::grafana_service::*;
pub use self::jaeger_service::*;
pub use self::meta_node_service::*;
Expand Down
68 changes: 68 additions & 0 deletions rust/risedev/src/task/frontend_service_v2.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
use std::env;
use std::path::Path;
use std::process::Command;

use anyhow::{anyhow, Result};

use super::{ExecuteContext, Task};
use crate::FrontendConfig;

pub struct FrontendServiceV2 {
config: FrontendConfig,
}

impl FrontendServiceV2 {
pub fn new(config: FrontendConfig) -> Result<Self> {
Ok(Self { config })
}

fn frontend_v2(&self) -> Result<Command> {
let prefix_bin = env::var("PREFIX_BIN")?;
Ok(Command::new(Path::new(&prefix_bin).join("frontend-v2")))
}
}

impl Task for FrontendServiceV2 {
fn execute(&mut self, ctx: &mut ExecuteContext<impl std::io::Write>) -> anyhow::Result<()> {
ctx.service(self);
ctx.pb.set_message("starting...");

let mut cmd = self.frontend_v2()?;

cmd.arg("--host")
.arg(format!("{}:{}", self.config.address, self.config.port));

let provide_meta_node = self.config.provide_meta_node.as_ref().unwrap();
match provide_meta_node.len() {
0 => {
return Err(anyhow!(
"Cannot start node: no meta node found in this configuration."
));
}
1 => {
let meta_node = &provide_meta_node[0];
cmd.arg("--meta-addr")
.arg(format!("http://{}:{}", meta_node.address, meta_node.port));
}
other_size => {
return Err(anyhow!(
"Cannot start node: {} meta nodes found in this configuration, but only 1 is needed.",
other_size
));
}
};

if !self.config.user_managed {
ctx.run_command(ctx.tmux_run(cmd)?)?;
ctx.pb.set_message("started");
} else {
ctx.pb.set_message("user managed");
}

Ok(())
}

fn id(&self) -> String {
self.config.id.clone()
}
}
2 changes: 1 addition & 1 deletion rust/utils/pgwire/src/pg_protocol.rs
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ where
}

async fn do_process(&mut self) -> Result<bool> {
let msg = self.read_message().await.unwrap();
let msg = self.read_message().await?;
match msg {
FeMessage::Ssl => {
self.write_message_no_flush(&BeMessage::EncryptionResponse)?;
Expand Down
Loading