From f966046af0e4a02302d7af73261f0ca3353d1ca9 Mon Sep 17 00:00:00 2001 From: "Felipe S. S. Schneider" Date: Fri, 12 May 2023 14:17:24 -0300 Subject: [PATCH 1/9] revert: start a complete rewrite --- Cargo.toml | 6 - answer/Cargo.toml | 25 ---- answer/README.md | 110 --------------- answer/src/main.rs | 343 --------------------------------------------- murmur/Cargo.toml | 26 ---- murmur/README.md | 5 - murmur/src/main.rs | 203 --------------------------- reply/Cargo.toml | 23 --- reply/README.md | 110 --------------- reply/src/main.rs | 225 ----------------------------- 10 files changed, 1076 deletions(-) delete mode 100644 Cargo.toml delete mode 100644 answer/Cargo.toml delete mode 100644 answer/README.md delete mode 100644 answer/src/main.rs delete mode 100644 murmur/Cargo.toml delete mode 100644 murmur/README.md delete mode 100644 murmur/src/main.rs delete mode 100644 reply/Cargo.toml delete mode 100644 reply/README.md delete mode 100644 reply/src/main.rs diff --git a/Cargo.toml b/Cargo.toml deleted file mode 100644 index 5b87e3a..0000000 --- a/Cargo.toml +++ /dev/null @@ -1,6 +0,0 @@ -[workspace] -members = ["answer", "murmur", "reply"] - -[profile.release] -lto = true -strip = true diff --git a/answer/Cargo.toml b/answer/Cargo.toml deleted file mode 100644 index fa8366f..0000000 --- a/answer/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "answer" -version = "0.0.1-beta.1" -edition = "2021" -authors = ["Felipe S. S. Schneider "] -description = "answer any question right from your terminal, using the same large language model that powers ChatGPT" -homepage = "https://github.com/schneiderfelipe/getanswe.rs/tree/main/answer" -repository = "https://github.com/schneiderfelipe/getanswe.rs" -license = "MIT" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -anyhow = { version = "1.0.71" } -async-openai = { version = "0.10.3" } -clap = { version = "4.2.7" } -clap-verbosity-flag = { version = "2.0.1" } -futures = { version = "0.3.28" } -human-panic = { version = "1.1.4" } -log = { version = "0.4.17" } -pretty_env_logger = { version = "0.4.0" } -serde = { version = "1.0.163" } -serde_yaml = { version = "0.9.21" } -thiserror = { version = "1.0.40" } -tokio = { version = "1.28.1", features = ["io-std", "rt-multi-thread"] } diff --git a/answer/README.md b/answer/README.md deleted file mode 100644 index 8d86525..0000000 --- a/answer/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# answer - -[![Crates.io version](https://img.shields.io/crates/v/answer)](https://crates.io/crates/answer) -[![GitHub license](https://img.shields.io/github/license/schneiderfelipe/getanswe.rs)](https://github.com/schneiderfelipe/getanswe.rs/blob/main/LICENSE) -[![Build CI](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/ci.yml/badge.svg)](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/ci.yml) -[![Changelog CI](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/changelog.yml/badge.svg)](https://github.com/schneiderfelipe/getanswe.rs/blob/main/CHANGELOG.md#changelog) -[![Libraries.io `SourceRank`](https://img.shields.io/librariesio/sourcerank/cargo/answer)](https://libraries.io/cargo/answer) - -> [`answer`💭](https://crates.io/crates/answer) _any_ question right from your terminal, -> using the same -> [large language model](https://en.wikipedia.org/wiki/Large_language_model) -> that powers -> [**`ChatGPT`**](https://chat.openai.com/chat). - -```console -$ echo "🌭 = 🥪?" | answer -No, a hot dog (🌭) is not the same as a sandwich (🥪). -While they both consist of bread and a filling, -a sandwich typically has separate slices of bread, -while a hot dog has a single bun that is sliced -on the top and filled with a sausage. -``` - -Read -the [installation](#installation) -and [usage](#usage) instructions below. - -### Installation - -#### From source (recommended) - -Either clone the repository to your machine and install from it, -or install directly from GitHub. -Both options require [Rust and Cargo to be installed](https://rustup.rs/). - -```console -# Option 1: cloning and installing from the repository -$ git clone https://github.com/schneiderfelipe/getanswe.rs.git -$ cd getanswe.rs && cargo install answer --path=answer/ - -# Option 2: installing directly from GitHub -$ cargo install answer --git=https://github.com/schneiderfelipe/getanswe.rs -``` - -If you're looking to contribute to the project's development, -the first option is the way to go (and thank you for your interest!). -However, -if you simply want to install the development version, -the second option is likely the better choice. - -### Environment Setup - -Before using [`answer`💭](https://crates.io/crates/answer), -you need to set up your environment to use -[`OpenAI`'s chat completion API](https://platform.openai.com/docs/guides/chat/chat-completions-beta) -(the same technology that powers `OpenAI`'s most advanced language model, -[`ChatGPT`](https://chat.openai.com/chat)). -To set up your environment, -you'll need to have a secret API key from `OpenAI`, -which can be obtained at -[`OpenAI`'s online platform](https://platform.openai.com/account/api-keys). - -Next, -set an environment variable in your shell as follows: - -```shell -export OPENAI_API_KEY="sk-...a1b2" -``` - -### Usage - -With your environment set up, -you're ready to start using -the command-line application. -Here's an example: - -```console -$ echo "Date of birth of Malcolm X?" | answer -The date of birth of Malcolm X is May 19, 1925. -``` - -You can also get `answer`s in context by providing a YAML file containing -the initial part of a chat history. -For example: - -```yaml -# birthdates.yml -messages: - - role: system - content: >- - You are a date of birth checker. - Given the name of a person, - your job is to specify the date of birth of said person. -``` - -```console -$ echo "Malcolm X" | answer birthdates.yml -Malcolm X was born on May 19th, 1925. -``` - -The file format closely resembles both -[`OpenAI`'s higher-level API](https://platform.openai.com/docs/guides/chat/introduction) -and -[its lower-level `ChatML` format](https://github.com/openai/openai-python/blob/main/chatml.md). - -### Unsafe code usage - -This project forbids unsafe code usage. - -License: MIT diff --git a/answer/src/main.rs b/answer/src/main.rs deleted file mode 100644 index 60eca1e..0000000 --- a/answer/src/main.rs +++ /dev/null @@ -1,343 +0,0 @@ -//! [![Crates.io version](https://img.shields.io/crates/v/answer)](https://crates.io/crates/answer) -//! [![GitHub license](https://img.shields.io/github/license/schneiderfelipe/getanswe.rs)](https://github.com/schneiderfelipe/getanswe.rs/blob/main/LICENSE) -//! [![Build CI](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/ci.yml/badge.svg)](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/ci.yml) -//! [![Changelog CI](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/changelog.yml/badge.svg)](https://github.com/schneiderfelipe/getanswe.rs/blob/main/CHANGELOG.md#changelog) -//! [![Libraries.io `SourceRank`](https://img.shields.io/librariesio/sourcerank/cargo/answer)](https://libraries.io/cargo/answer) -//! -//! > [`answer`💭](https://crates.io/crates/answer) _any_ question right from -//! > your terminal, -//! > using the same -//! > [large language model](https://en.wikipedia.org/wiki/Large_language_model) -//! > that powers -//! > [**`ChatGPT`**](https://chat.openai.com/chat). -//! -//! ```console -//! $ echo "🌭 = 🥪?" | answer -//! No, a hot dog (🌭) is not the same as a sandwich (🥪). -//! While they both consist of bread and a filling, -//! a sandwich typically has separate slices of bread, -//! while a hot dog has a single bun that is sliced -//! on the top and filled with a sausage. -//! ``` -//! -//! Read -//! the [installation](#installation) -//! and [usage](#usage) instructions below. -//! -//! ## Installation -//! -//! ### From source (recommended) -//! -//! Either clone the repository to your machine and install from it, -//! or install directly from GitHub. -//! Both options require [Rust and Cargo to be installed](https://rustup.rs/). -//! -//! ```console -//! # Option 1: cloning and installing from the repository -//! $ git clone https://github.com/schneiderfelipe/getanswe.rs.git -//! $ cd getanswe.rs && cargo install answer --path=answer/ -//! -//! # Option 2: installing directly from GitHub -//! $ cargo install answer --git=https://github.com/schneiderfelipe/getanswe.rs -//! ``` -//! -//! If you're looking to contribute to the project's development, -//! the first option is the way to go (and thank you for your interest!). -//! However, -//! if you simply want to install the development version, -//! the second option is likely the better choice. -//! -//! ## Environment Setup -//! -//! Before using [`answer`💭](https://crates.io/crates/answer), -//! you need to set up your environment to use -//! [`OpenAI`'s chat completion API](https://platform.openai.com/docs/guides/chat/chat-completions-beta) -//! (the same technology that powers `OpenAI`'s most advanced language model, -//! [`ChatGPT`](https://chat.openai.com/chat)). -//! To set up your environment, -//! you'll need to have a secret API key from `OpenAI`, -//! which can be obtained at -//! [`OpenAI`'s online platform](https://platform.openai.com/account/api-keys). -//! -//! Next, -//! set an environment variable in your shell as follows: -//! -//! ```shell -//! export OPENAI_API_KEY="sk-...a1b2" -//! ``` -//! -//! ## Usage -//! -//! With your environment set up, -//! you're ready to start using -//! the command-line application. -//! Here's an example: -//! -//! ```console -//! $ echo "Date of birth of Malcolm X?" | answer -//! The date of birth of Malcolm X is May 19, 1925. -//! ``` -//! -//! You can also get `answer`s in context by providing a YAML file containing -//! the initial part of a chat history. -//! For example: -//! -//! ```yaml -//! # birthdates.yml -//! messages: -//! - role: system -//! content: >- -//! You are a date of birth checker. -//! Given the name of a person, -//! your job is to specify the date of birth of said person. -//! ``` -//! -//! ```console -//! $ echo "Malcolm X" | answer birthdates.yml -//! Malcolm X was born on May 19th, 1925. -//! ``` -//! -//! The file format closely resembles both -//! [`OpenAI`'s higher-level API](https://platform.openai.com/docs/guides/chat/introduction) -//! and -//! [its lower-level `ChatML` format](https://github.com/openai/openai-python/blob/main/chatml.md). -//! -//! ## Unsafe code usage -//! -//! This project forbids unsafe code usage. - -#![forbid(unsafe_code)] - -use std::env; -use std::fs::File; -use std::io::Read; -use std::io::{self}; - -use async_openai::error::OpenAIError; -use async_openai::types::ChatCompletionRequestMessage; -use async_openai::types::CreateChatCompletionRequestArgs; -use async_openai::types::Role; -use async_openai::Client; -use clap::Parser; -use futures::StreamExt; -use serde::Deserialize; -use serde::Serialize; -use thiserror::Error; -use tokio::io::AsyncReadExt; -use tokio::io::AsyncWrite; -use tokio::io::AsyncWriteExt; - -/// The context of a conversation. -/// -/// It can be used for building prompts or storing chat history. -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -struct Conversation { - /// [`Message`]s in this [`Conversation`]. - #[serde(default, skip_serializing_if = "Vec::is_empty")] - messages: Vec, -} - -impl Conversation { - /// Append a new [`Message`] to the end of this [`Conversation`]. - #[inline] - fn push(&mut self, message: Message) { - self.messages.push(message); - } - - /// Parse a [`Conversation`] from a [`Read`]er. - #[inline] - fn from_reader(reader: R) -> Result - where - R: Read, - { - serde_yaml::from_reader(reader) - } -} - -/// A [`Conversation`] message. -/// -/// This is basically a redefinition of [`ChatCompletionRequestMessage`] -/// so that we can implement new traits and methods. -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -struct Message { - /// The [`Role`] of the author of the [`Message`]. - #[serde(default, skip_serializing_if = "is_user")] - role: Role, - /// The contents of the [`Message`]. - #[serde(default, skip_serializing_if = "String::is_empty")] - content: String, - /// The name of the author in a multi-agent [`Conversation`]. - #[serde(default, skip_serializing_if = "Option::is_none")] - name: Option, -} - -impl Message { - /// Create a [`Message`] whose [`Role`] is user. - #[inline] - fn from_user(content: C) -> Self - where - C: Into, - { - Self { - role: Role::User, - content: content.into(), - name: None, - } - } -} - -impl From for ChatCompletionRequestMessage { - /// Convert a [`Message`] into a [`ChatCompletionRequestMessage`]. - #[inline] - fn from(message: Message) -> Self { - Self { - role: message.role, - content: message.content, - name: message.name, - } - } -} - -/// A robot that answers questions in plain text. -#[derive(Debug, Default, Serialize, Deserialize)] -struct Bot {} - -/// An error that came from [`Bot`]. -#[derive(Debug, Error)] -enum BotError { - #[error("could not obtain environment variable: {0}")] - Var(#[from] env::VarError), - #[error("could not exchange data with OpenAI: {0}")] - OpenAI(#[from] OpenAIError), - #[error("could not perform an input or output operation: {0}")] - Io(#[from] io::Error), -} - -impl Bot { - /// Reply, in the context of a [`Conversation`], to the given - /// [`AsyncWrite`]r. - #[inline] - async fn reply_to_writer( - &self, - conversation: &Conversation, - mut writer: W, - ) -> Result<(), BotError> - where - W: AsyncWrite + Send + Unpin, - { - let mut stream = Client::default() - .with_api_key(env::var("OPENAI_API_KEY")?) - .chat() - .create_stream({ - CreateChatCompletionRequestArgs::default() - .model("gpt-3.5-turbo") - .temperature(0.0) - .messages( - conversation - .messages - .iter() - .cloned() - .map(Into::into) - .collect::>(), - ) - .build()? - }) - .await?; - - while let Some(response) = stream.next().await { - for content in response? - .choices - .into_iter() - .filter_map(|choice| choice.delta.content) - { - writer.write_all(content.as_bytes()).await?; - } - - writer.flush().await?; - } - - Ok(()) - } -} - -/// answer any question right from your terminal, -/// using the same large language model that powers `ChatGPT`. -/// -/// The program takes in user messages from the standard input -/// and outputs assistant messages to the standard output. -#[derive(Debug, Parser)] -#[command(author, version, about)] -#[command(propagate_version = true)] -struct Cli { - /// Path to a conversation YAML file. - #[arg(value_parser = parse_conversation)] - conversation: Option, - - /// Verbosity options. - #[clap(flatten)] - verbosity: clap_verbosity_flag::Verbosity, -} - -/// An error that came from [`Cli`]. -#[derive(Debug, Error)] -enum CliError { - #[error("could not perform a serialization or deserialization operation: {0}")] - Yaml(#[from] serde_yaml::Error), - #[error("could not perform an input or output operation: {0}")] - Io(#[from] io::Error), -} - -/// Get a [`Conversation`] from a file [`Path`] by parsing. -#[inline] -fn parse_conversation(path: &str) -> Result { - let file = File::open(path)?; - let conversation = Conversation::from_reader(file)?; - Ok(conversation) -} - -/// Our beloved main function. -#[tokio::main] -async fn main() -> anyhow::Result<()> { - human_panic::setup_panic!(); - - let cli = Cli::parse(); - pretty_env_logger::formatted_builder() - .filter_level(cli.verbosity.log_level_filter()) - .init(); - log::debug!("{cli:#?}"); - - let mut conversation = cli.conversation.unwrap_or_default(); - - conversation.push({ - let mut content = String::new(); - tokio::io::stdin().read_to_string(&mut content).await?; - - Message::from_user(content) - }); - - Bot::default() - .reply_to_writer(&conversation, tokio::io::stdout()) - .await?; - Ok(()) -} - -/// Determine whether a [`Role`] corresponds to a user. -#[inline] -const fn is_user(role: &Role) -> bool { - match role { - Role::User => true, - Role::System | Role::Assistant => false, - } -} - -#[cfg(test)] -mod tests { - use clap::CommandFactory; - - use super::*; - - #[test] - fn verify_cli() { - Cli::command().debug_assert(); - } -} diff --git a/murmur/Cargo.toml b/murmur/Cargo.toml deleted file mode 100644 index 2a5b153..0000000 --- a/murmur/Cargo.toml +++ /dev/null @@ -1,26 +0,0 @@ -[package] -name = "murmur" -version = "0.0.0" -edition = "2021" -authors = ["Felipe S. S. Schneider "] -description = "murmur into your terminal and convert your speech to text using OpenAI's Whisper API" -homepage = "https://github.com/schneiderfelipe/getanswe.rs/tree/main/murmur" -repository = "https://github.com/schneiderfelipe/getanswe.rs" -license = "MIT" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -anyhow = { version = "1.0.71" } -async-openai = { version = "0.9.5" } -clap = { version = "4.2.7", features = ["derive"] } -clap-verbosity-flag = { version = "2.0.1" } -cpal = { version = "0.15.2" } -ctrlc = "3.2.5" -either = { version = "1.8.1" } -hound = { version = "3.5.0" } -human-panic = { version = "1.1.4" } -log = { version = "0.4.17" } -pretty_env_logger = { version = "0.4.0" } -tempfile = { version = "3.5.0" } -tokio = { version = "1.28.1", features = ["rt-multi-thread"] } diff --git a/murmur/README.md b/murmur/README.md deleted file mode 100644 index 9dd1b5c..0000000 --- a/murmur/README.md +++ /dev/null @@ -1,5 +0,0 @@ -# murmur - -murmur transcribes speech to text from the command-line through OpenAI's Whisper API. - -License: MIT diff --git a/murmur/src/main.rs b/murmur/src/main.rs deleted file mode 100644 index 6d80d03..0000000 --- a/murmur/src/main.rs +++ /dev/null @@ -1,203 +0,0 @@ -//! murmur into your terminal and convert your speech to text using `OpenAI`'s -//! Whisper API. -//! -//! Records a WAV file using the default input device and config until the user -//! indicates end of input. -//! -//! The input data is recorded to "$`CARGO_MANIFEST_DIR/recorded.wav`". -//! -//! ## Installation -//! -//! Note: if you're using [ALSA](https://www.alsa-project.org/wiki/Main_Page), -//! you may need to install the development files for `libasound2`, -//! -//! ```console -//! $ sudo apt install libasound2-dev -//! ``` - -#![forbid(unsafe_code)] - -use std::env; -use std::fs::File; -use std::io::BufWriter; -use std::io::Write; -use std::io::{self}; -use std::iter::once; -use std::sync::mpsc::channel; -use std::sync::Arc; -use std::sync::Mutex; - -use async_openai::types::CreateTranscriptionRequestArgs; -use async_openai::Client; -use clap::Parser; -use cpal::traits::DeviceTrait; -use cpal::traits::HostTrait; -use cpal::traits::StreamTrait; -use cpal::FromSample; -use cpal::Sample; -use either::Either; - -/// murmur into your terminal and convert your speech to text using `OpenAI`'s -/// Whisper API. -/// -/// The program will continue recording until you signal "end of file" (Ctrl-D), -/// and then it will output the transcribed text to the standard output. -#[derive(Debug, Parser)] -#[command(author, version, about)] -#[command(propagate_version = true)] -struct Cli { - /// Verbosity options. - #[clap(flatten)] - verbosity: clap_verbosity_flag::Verbosity, -} - -/// Our beloved main function. -#[tokio::main] -async fn main() -> anyhow::Result<()> { - human_panic::setup_panic!(); - - let cli = Cli::parse(); - pretty_env_logger::formatted_builder() - .filter_level(cli.verbosity.log_level_filter()) - .init(); - log::debug!("{cli:#?}"); - - let host = cpal::default_host(); - - // Set up the input devices and stream with the default input configs. - let devices = host.input_devices()?; - let devices = if let Some(device) = host.default_input_device() { - Either::Left(once(device).chain(devices)) - } else { - Either::Right(devices) - }; - - let mut devices_configs = devices.filter_map(|device| { - device - .default_input_config() - .ok() - .map(|config| (device, config)) - }); - - let Some((device, config)) = devices_configs.next() else { - anyhow::bail!("Failed to get default input config"); - }; - - // The WAV file we're recording to. - let path = tempfile::Builder::new() - .prefix("murmur") - .suffix(".wav") - .tempfile()? - .into_temp_path(); - log::debug!("{path:#?}"); - let spec = wav_spec_from_config(&config)?; - let writer = hound::WavWriter::create(&path, spec)?; - let writer = Arc::new(Mutex::new(Some(writer))); - - // Run the input stream on a separate thread. - let writer_2 = writer.clone(); - - let err_fn = move |err| { - log::error!("an error occurred on stream: {err}"); - }; - - let stream = match config.sample_format() { - cpal::SampleFormat::I8 => { - device.build_input_stream( - &config.into(), - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )? - } - cpal::SampleFormat::I16 => { - device.build_input_stream( - &config.into(), - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )? - } - cpal::SampleFormat::I32 => { - device.build_input_stream( - &config.into(), - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )? - } - cpal::SampleFormat::F32 => { - device.build_input_stream( - &config.into(), - move |data, _: &_| write_input_data::(data, &writer_2), - err_fn, - None, - )? - } - sample_format => { - return Err(anyhow::Error::msg(format!( - "Unsupported sample format '{sample_format}'" - ))) - } - }; - - stream.play()?; - let (tx, rx) = channel(); - ctrlc::set_handler(move || tx.send(()).expect("should send signal on channel"))?; - - rx.recv()?; - drop(stream); - writer.lock().unwrap().take().unwrap().finalize()?; - - let response = Client::default() - .with_api_key(env::var("OPENAI_API_KEY")?) - .audio() - .transcribe( - CreateTranscriptionRequestArgs::default() - .model("whisper-1") - .file(&path) - .build()?, - ) - .await?; - - log::debug!("{response:#?}"); - writeln!(io::stdout().lock(), "{text}", text = response.text)?; - - path.close()?; - Ok(()) -} - -fn sample_format(format: cpal::SampleFormat) -> hound::SampleFormat { - if format.is_float() { - hound::SampleFormat::Float - } else { - hound::SampleFormat::Int - } -} - -fn wav_spec_from_config(config: &cpal::SupportedStreamConfig) -> anyhow::Result { - let wav_spec = hound::WavSpec { - channels: config.channels() as _, - sample_rate: config.sample_rate().0 as _, - bits_per_sample: u16::try_from(config.sample_format().sample_size() * 8)?, - sample_format: sample_format(config.sample_format()), - }; - Ok(wav_spec) -} - -type WavWriterHandle = Arc>>>>; - -fn write_input_data(input: &[T], writer: &WavWriterHandle) -where - T: Sample, - U: Sample + hound::Sample + FromSample, -{ - if let Ok(mut guard) = writer.try_lock() { - if let Some(writer) = guard.as_mut() { - for &sample in input.iter() { - let sample: U = U::from_sample(sample); - writer.write_sample(sample).ok(); - } - } - } -} diff --git a/reply/Cargo.toml b/reply/Cargo.toml deleted file mode 100644 index 0e0ec92..0000000 --- a/reply/Cargo.toml +++ /dev/null @@ -1,23 +0,0 @@ -[package] -name = "reply" -version = "0.0.1-beta.2" -edition = "2021" -authors = ["Felipe S. S. Schneider "] -description = "reply makes any command-line application a (stateless) REPL" -homepage = "https://github.com/schneiderfelipe/getanswe.rs/tree/main/reply" -repository = "https://github.com/schneiderfelipe/getanswe.rs" -license = "MIT" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -anyhow = { version = "1.0.71" } -clap = { version = "4.2.7" } -clap-verbosity-flag = { version = "2.0.1" } -duct = { version = "0.13.6" } -duct_sh = { version = "0.13.6" } -human-panic = { version = "1.1.4" } -log = { version = "0.4.17" } -pretty_env_logger = { version = "0.4.0" } -rustyline = { version = "11.0.0" } -thiserror = { version = "1.0.40" } diff --git a/reply/README.md b/reply/README.md deleted file mode 100644 index dd0d844..0000000 --- a/reply/README.md +++ /dev/null @@ -1,110 +0,0 @@ -# reply - -[![Crates.io version](https://img.shields.io/crates/v/reply)](https://crates.io/crates/reply) -[![GitHub license](https://img.shields.io/github/license/schneiderfelipe/getanswe.rs)](https://github.com/schneiderfelipe/getanswe.rs/blob/main/LICENSE) -[![Build CI](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/ci.yml/badge.svg)](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/ci.yml) -[![Changelog CI](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/changelog.yml/badge.svg)](https://github.com/schneiderfelipe/getanswe.rs/blob/main/CHANGELOG.md#changelog) -[![Libraries.io `SourceRank`](https://img.shields.io/librariesio/sourcerank/cargo/reply)](https://libraries.io/cargo/reply) - -> [`reply`📩](https://crates.io/crates/reply) makes any command-line application a (stateless) [REPL](https://en.wikipedia.org/wiki/Read%E2%80%93eval%E2%80%93print_loop). - -```console -$ reply 'python | cowsay -f tux -n' -> print("Hello reply📩!") - ________________ -< Hello reply📩! > - ---------------- - \ - \ - .--. - |o_o | - |:_/ | - // \ \ - (| | ) - /'\_ _/`\ - \___)=(___/ - -> -``` - -Read -the [installation](#installation) -and [usage](#usage) instructions below. - -### Installation - -#### From source (recommended) - -Either clone the repository to your machine and install from it, -or install directly from GitHub. -Both options require [Rust and Cargo to be installed](https://rustup.rs/). - -```console -# Option 1: cloning and installing from the repository -$ git clone https://github.com/schneiderfelipe/getanswe.rs.git -$ cd getanswe.rs && cargo install reply --path=reply/ - -# Option 2: installing directly from GitHub -$ cargo install reply --git=https://github.com/schneiderfelipe/getanswe.rs -``` - -If you're looking to contribute to the project's development, -the first option is the way to go (and thank you for your interest!). -However, -if you simply want to install the development version, -the second option is likely the better choice. - -### Usage - -Using this tool is simple: - -```console -$ reply 'python' -> -``` - -Whatever you type in the prompt will be fed to the backend command (`python` in the example). -The output of the command will be displayed in the terminal. -For example: - -```console -$ reply 'python' -> print("Hello " + "python") -Hello python -> -``` - -However, -there are a few things to keep in mind: - -- Only the standard output is captured. - If nothing is printed, - nothing will be shown. -- The REPL is stateless, - which means that there's no memory being carried out. - If you define a variable, - for example, - it won't be available in the next prompt. - -Here's an example: - -```console -$ reply 'python' -> a = 2 # no output -> print(f"a = {a}") # no memory -Traceback (most recent call last): - File "", line 1, in -NameError: name 'a' is not defined -``` - -Therefore, -it's the responsibility of the backend application to - -- Print out results to the standard output. -- Implement memory (normally through a file). - -### Unsafe code usage - -This project forbids unsafe code usage. - -License: MIT diff --git a/reply/src/main.rs b/reply/src/main.rs deleted file mode 100644 index 9888148..0000000 --- a/reply/src/main.rs +++ /dev/null @@ -1,225 +0,0 @@ -//! [![Crates.io version](https://img.shields.io/crates/v/reply)](https://crates.io/crates/reply) -//! [![GitHub license](https://img.shields.io/github/license/schneiderfelipe/getanswe.rs)](https://github.com/schneiderfelipe/getanswe.rs/blob/main/LICENSE) -//! [![Build CI](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/ci.yml/badge.svg)](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/ci.yml) -//! [![Changelog CI](https://github.com/schneiderfelipe/getanswe.rs/actions/workflows/changelog.yml/badge.svg)](https://github.com/schneiderfelipe/getanswe.rs/blob/main/CHANGELOG.md#changelog) -//! [![Libraries.io `SourceRank`](https://img.shields.io/librariesio/sourcerank/cargo/reply)](https://libraries.io/cargo/reply) -//! -//! > [`reply`📩](https://crates.io/crates/reply) makes any command-line -//! > application -//! > a (stateless) [REPL](https://en.wikipedia.org/wiki/Read%E2%80%93eval%E2%80%93print_loop). -//! -//! ```console -//! $ reply 'python | cowsay -f tux -n' -//! > print("Hello reply📩!") -//! ________________ -//! < Hello reply📩! > -//! ---------------- -//! \ -//! \ -//! .--. -//! |o_o | -//! |:_/ | -//! // \ \ -//! (| | ) -//! /'\_ _/`\ -//! \___)=(___/ -//! -//! > -//! ``` -//! -//! Read -//! the [installation](#installation) -//! and [usage](#usage) instructions below. -//! -//! ## Installation -//! -//! ### From source (recommended) -//! -//! Either clone the repository to your machine and install from it, -//! or install directly from GitHub. -//! Both options require [Rust and Cargo to be installed](https://rustup.rs/). -//! -//! ```console -//! # Option 1: cloning and installing from the repository -//! $ git clone https://github.com/schneiderfelipe/getanswe.rs.git -//! $ cd getanswe.rs && cargo install reply --path=reply/ -//! -//! # Option 2: installing directly from GitHub -//! $ cargo install reply --git=https://github.com/schneiderfelipe/getanswe.rs -//! ``` -//! -//! If you're looking to contribute to the project's development, -//! the first option is the way to go (and thank you for your interest!). -//! However, -//! if you simply want to install the development version, -//! the second option is likely the better choice. -//! -//! ## Usage -//! -//! Using this tool is simple: -//! -//! ```console -//! $ reply 'python' -//! > -//! ``` -//! -//! Whatever you type in the prompt will be fed to the backend command (`python` -//! in the example). The output of the command will be displayed in the -//! terminal. For example: -//! -//! ```console -//! $ reply 'python' -//! > print("Hello " + "python") -//! Hello python -//! > -//! ``` -//! -//! However, -//! there are a few things to keep in mind: -//! -//! - Only the standard output is captured. If nothing is printed, nothing will -//! be shown. -//! - The REPL is stateless, which means that there's no memory being carried -//! out. If you define a variable, for example, it won't be available in the -//! next prompt. -//! -//! Here's an example: -//! -//! ```console -//! $ reply 'python' -//! > a = 2 # no output -//! > print(f"a = {a}") # no memory -//! Traceback (most recent call last): -//! File "", line 1, in -//! NameError: name 'a' is not defined -//! ``` -//! -//! Therefore, -//! it's the responsibility of the backend application to -//! -//! - Print out results to the standard output. -//! - Implement memory (normally through a file). -//! -//! ## Unsafe code usage -//! -//! This project forbids unsafe code usage. - -#![forbid(unsafe_code)] - -use std::env; -use std::io::Read; -use std::io::Write; -use std::io::{self}; - -use clap::Parser; -use duct::Expression; -use duct_sh::sh_dangerous; -use rustyline::error::ReadlineError; -use rustyline::Cmd; -use rustyline::Config; -use rustyline::Editor; -use rustyline::KeyEvent; -use thiserror::Error; - -/// reply makes any command-line application a (stateless) REPL. -/// -/// This program sets up a REPL (Read-Evaluate-Print Loop) -/// that takes user input -/// and sends it to the backend application's standard input for evaluation. -/// The output content is retrieved from the application's standard output -/// and printed. -/// This loop continues until the program is terminated. -#[derive(Debug, Parser)] -#[command(author, version, about)] -#[command(propagate_version = true)] -struct Cli { - /// The expression that will run as the backend application - /// when user input is received. - #[arg(value_parser = parse_expression)] - expression: Expression, - - /// Verbosity options. - #[clap(flatten)] - verbosity: clap_verbosity_flag::Verbosity, -} - -/// An error that came from [`Cli`]. -#[derive(Debug, Error)] -enum CliError {} - -/// Get an [`Expression`] by parsing. -#[allow(clippy::unnecessary_wraps)] -#[inline] -fn parse_expression(input: &str) -> Result { - let expression = sh_dangerous(input); - Ok(expression) -} - -/// Our beloved main function. -fn main() -> anyhow::Result<()> { - human_panic::setup_panic!(); - - let cli = Cli::parse(); - pretty_env_logger::formatted_builder() - .filter_level(cli.verbosity.log_level_filter()) - .init(); - log::debug!("{cli:#?}"); - - let mut editor = Editor::with_config(Config::builder().auto_add_history(true).build())?; - editor.set_helper(Some(())); - editor.bind_sequence(KeyEvent::alt('\r'), Cmd::Newline); - - // let history_path = data_dir.join("history.txt"); - // if editor.load_history(&history_path).is_err() { - // log::warn!("No previous history found."); - // } - - // Read until the line has at least one non-whitespace character. - let mut readline = || { - loop { - let line = editor.readline("> "); - match line { - Ok(ref l) if !l.trim().is_empty() => break line, - err @ Err(_) => break err, - _ => {} - } - } - }; - - loop { - match readline() { - Ok(line) => { - // editor.save_history(&self.history_path)?; - - let mut output = String::new(); - cli.expression - .unchecked() - .stdin_bytes(line) - .reader()? - .read_to_string(&mut output)?; - - let mut stdout = io::stdout().lock(); - writeln!(stdout, "{output}")?; - stdout.flush()?; - } - Err(ReadlineError::Interrupted | ReadlineError::Eof) => break, - err @ Err(_) => { - err?; - } - } - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - use clap::CommandFactory; - - use super::*; - - #[test] - fn verify_cli() { - Cli::command().debug_assert(); - } -} From 48ce8d7723026809f5055c769b84de646be31a10 Mon Sep 17 00:00:00 2001 From: "Felipe S. S. Schneider" Date: Fri, 12 May 2023 14:24:57 -0300 Subject: [PATCH 2/9] feat: start the new answer package --- .github/workflows/CI (Case Conflict).yml | 120 +++++++++++++++++++++++ Cargo.toml | 12 +++ pyproject.toml | 17 ++++ python/answer/__init__.py | 6 ++ src/lib.rs | 14 +++ 5 files changed, 169 insertions(+) create mode 100644 .github/workflows/CI (Case Conflict).yml create mode 100644 Cargo.toml create mode 100644 pyproject.toml create mode 100644 python/answer/__init__.py create mode 100644 src/lib.rs diff --git a/.github/workflows/CI (Case Conflict).yml b/.github/workflows/CI (Case Conflict).yml new file mode 100644 index 0000000..a1a573a --- /dev/null +++ b/.github/workflows/CI (Case Conflict).yml @@ -0,0 +1,120 @@ +# This file is autogenerated by maturin v0.15.1 +# To update, run +# +# maturin generate-ci github +# +name: CI + +on: + push: + branches: + - main + - master + tags: + - '*' + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + linux: + runs-on: ubuntu-latest + strategy: + matrix: + target: [x86_64, x86, aarch64, armv7, s390x, ppc64le] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + manylinux: auto + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + windows: + runs-on: windows-latest + strategy: + matrix: + target: [x64, x86] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + architecture: ${{ matrix.target }} + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + macos: + runs-on: macos-latest + strategy: + matrix: + target: [x86_64, aarch64] + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.10' + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.target }} + args: --release --out dist --find-interpreter + sccache: 'true' + - name: Upload wheels + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - name: Upload sdist + uses: actions/upload-artifact@v3 + with: + name: wheels + path: dist + + release: + name: Release + runs-on: ubuntu-latest + if: "startsWith(github.ref, 'refs/tags/')" + needs: [linux, windows, macos, sdist] + steps: + - uses: actions/download-artifact@v3 + with: + name: wheels + - name: Publish to PyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + with: + command: upload + args: --skip-existing * diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..47c0fce --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "answer" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[lib] +name = "answer" +crate-type = ["cdylib"] + +[dependencies] +pyo3 = "0.18.3" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..257b9d5 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,17 @@ +[build-system] +requires = ["maturin>=0.15,<0.16"] +build-backend = "maturin" + +[project] +name = "answer" +requires-python = ">=3.7" +classifiers = [ + "Programming Language :: Rust", + "Programming Language :: Python :: Implementation :: CPython", + "Programming Language :: Python :: Implementation :: PyPy", +] + + +[tool.maturin] +python-source = "python" +features = ["pyo3/extension-module"] diff --git a/python/answer/__init__.py b/python/answer/__init__.py new file mode 100644 index 0000000..6f6d361 --- /dev/null +++ b/python/answer/__init__.py @@ -0,0 +1,6 @@ +from .answer import * # noqa: F403 + + +__doc__ = answer.__doc__ # noqa: F405 +if hasattr(answer, "__all__"): # noqa: F405 + __all__ = answer.__all__ # noqa: F405 diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..c482611 --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,14 @@ +use pyo3::prelude::*; + +/// Formats the sum of two numbers as string. +#[pyfunction] +fn sum_as_string(a: usize, b: usize) -> PyResult { + Ok((a + b).to_string()) +} + +/// A Python module implemented in Rust. +#[pymodule] +fn answer(_py: Python, m: &PyModule) -> PyResult<()> { + m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; + Ok(()) +} From 11120127e1213391983cc64c86fb744467ca83cd Mon Sep 17 00:00:00 2001 From: "Felipe S. S. Schneider" Date: Fri, 12 May 2023 14:28:26 -0300 Subject: [PATCH 3/9] ci: update workflows --- .../workflows/{CI (Case Conflict).yml => CI.yml} | 14 +++++++------- .github/workflows/{ci.yml => rust.yml} | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) rename .github/workflows/{CI (Case Conflict).yml => CI.yml} (93%) rename .github/workflows/{ci.yml => rust.yml} (99%) diff --git a/.github/workflows/CI (Case Conflict).yml b/.github/workflows/CI.yml similarity index 93% rename from .github/workflows/CI (Case Conflict).yml rename to .github/workflows/CI.yml index a1a573a..19cd857 100644 --- a/.github/workflows/CI (Case Conflict).yml +++ b/.github/workflows/CI.yml @@ -11,7 +11,7 @@ on: - main - master tags: - - '*' + - "*" pull_request: workflow_dispatch: @@ -28,13 +28,13 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: "3.10" - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} args: --release --out dist --find-interpreter - sccache: 'true' + sccache: "true" manylinux: auto - name: Upload wheels uses: actions/upload-artifact@v3 @@ -51,14 +51,14 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: "3.10" architecture: ${{ matrix.target }} - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} args: --release --out dist --find-interpreter - sccache: 'true' + sccache: "true" - name: Upload wheels uses: actions/upload-artifact@v3 with: @@ -74,13 +74,13 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: "3.10" - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} args: --release --out dist --find-interpreter - sccache: 'true' + sccache: "true" - name: Upload wheels uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/ci.yml b/.github/workflows/rust.yml similarity index 99% rename from .github/workflows/ci.yml rename to .github/workflows/rust.yml index 34eee22..5df2c9f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/rust.yml @@ -1,4 +1,4 @@ -name: CI +name: Rust on: push: From 3c53505f8c69bfd7990d23c6b4c4c765bda09b9e Mon Sep 17 00:00:00 2001 From: "Felipe S. S. Schneider" Date: Fri, 12 May 2023 17:09:11 -0300 Subject: [PATCH 4/9] chore(.gitignore): update with Python and Rust --- .gitignore | 179 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 175 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 389159a..a0186ef 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,182 @@ -# Created by https://www.toptal.com/developers/gitignore/api/rust -# Edit at https://www.toptal.com/developers/gitignore?templates=rust +# Created by https://www.toptal.com/developers/gitignore/api/python,rust +# Edit at https://www.toptal.com/developers/gitignore?templates=python,rust + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json ### Rust ### # Generated by Cargo # will have compiled files and executables debug/ -target/ # Remove Cargo.lock from gitignore if creating an executable, leave it for libraries # More information here https://doc.rust-lang.org/cargo/guide/cargo-toml-vs-cargo-lock.html @@ -17,4 +188,4 @@ Cargo.lock # MSVC Windows builds of rustc generate these, which store debugging information *.pdb -# End of https://www.toptal.com/developers/gitignore/api/rust +# End of https://www.toptal.com/developers/gitignore/api/python,rust From 9986e9a1c569403c747edb063c554458693a4610 Mon Sep 17 00:00:00 2001 From: "Felipe S. S. Schneider" Date: Fri, 12 May 2023 17:14:09 -0300 Subject: [PATCH 5/9] revert: move to a bin maturin project --- Cargo.toml | 4 ---- pyproject.toml | 2 +- src/lib.rs | 14 -------------- src/main.rs | 3 +++ 4 files changed, 4 insertions(+), 19 deletions(-) delete mode 100644 src/lib.rs create mode 100644 src/main.rs diff --git a/Cargo.toml b/Cargo.toml index 47c0fce..fb20d09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,9 +4,5 @@ version = "0.1.0" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html -[lib] -name = "answer" -crate-type = ["cdylib"] [dependencies] -pyo3 = "0.18.3" diff --git a/pyproject.toml b/pyproject.toml index 257b9d5..ad6d875 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -13,5 +13,5 @@ classifiers = [ [tool.maturin] +bindings = "bin" python-source = "python" -features = ["pyo3/extension-module"] diff --git a/src/lib.rs b/src/lib.rs deleted file mode 100644 index c482611..0000000 --- a/src/lib.rs +++ /dev/null @@ -1,14 +0,0 @@ -use pyo3::prelude::*; - -/// Formats the sum of two numbers as string. -#[pyfunction] -fn sum_as_string(a: usize, b: usize) -> PyResult { - Ok((a + b).to_string()) -} - -/// A Python module implemented in Rust. -#[pymodule] -fn answer(_py: Python, m: &PyModule) -> PyResult<()> { - m.add_function(wrap_pyfunction!(sum_as_string, m)?)?; - Ok(()) -} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} From dcdc290994adeb4c7c5e168c4b25140cb0441046 Mon Sep 17 00:00:00 2001 From: "Felipe S. S. Schneider" Date: Fri, 12 May 2023 18:24:14 -0300 Subject: [PATCH 6/9] feat: reimplement reply features --- Cargo.toml | 7 +++++ src/main.rs | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index fb20d09..f274585 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,3 +6,10 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +anyhow = "1.0.71" +clap = { version = "4.2.7", features = ["derive"] } +clap-verbosity-flag = "2.0.1" +human-panic = "1.1.4" +log = "0.4.17" +pretty_env_logger = "0.4.0" +rustyline = "11.0.0" diff --git a/src/main.rs b/src/main.rs index e7a11a9..8ad6f1b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,3 +1,79 @@ -fn main() { - println!("Hello, world!"); +#![forbid(unsafe_code)] + +use std::io; +use std::io::Write; + +use clap::Parser; +use rustyline::error::ReadlineError; +use rustyline::history::History; +use rustyline::Cmd; +use rustyline::Config; +use rustyline::Editor; +use rustyline::Helper; +use rustyline::KeyEvent; + +#[derive(Debug, Parser)] +#[command(author, version, about)] +#[command(propagate_version = true)] +struct Cli { + #[clap(flatten)] + verbosity: clap_verbosity_flag::Verbosity, +} + +fn main() -> anyhow::Result<()> { + human_panic::setup_panic!(); + + let cli = Cli::parse(); + pretty_env_logger::formatted_builder() + .filter_level(cli.verbosity.log_level_filter()) + .init(); + log::debug!("{cli:#?}"); + + let mut editor = Editor::with_config(Config::builder().auto_add_history(true).build())?; + editor.set_helper(Some(())); + editor.bind_sequence(KeyEvent::alt('\r'), Cmd::Newline); + + loop { + match read_line(&mut editor) { + Ok(line) => process_line(&line)?, + Err(ReadlineError::Interrupted | ReadlineError::Eof) => break, + err @ Err(_) => { + err?; + } + } + } + + Ok(()) +} + +#[inline] +fn read_line(editor: &mut Editor) -> rustyline::Result { + loop { + let line = editor.readline("💬 "); + match line { + Ok(ref l) if !l.trim().is_empty() => break line, + err @ Err(_) => break err, + _ => {} + } + } +} + +#[inline] +fn process_line(line: &str) -> io::Result<()> { + let mut stdout = io::stdout().lock(); + writeln!(stdout, "GOT: {line}")?; + stdout.flush()?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use clap::CommandFactory; + + use super::*; + + #[test] + fn verify_cli() { + Cli::command().debug_assert(); + } } From 84a56b158f5a9d4d2e0b47638a72ec21de654c26 Mon Sep 17 00:00:00 2001 From: "Felipe S. S. Schneider" Date: Sun, 14 May 2023 12:48:52 -0300 Subject: [PATCH 7/9] feat: add the machinery for recording audio --- Cargo.toml | 7 +++ src/main.rs | 170 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 172 insertions(+), 5 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f274585..5f38c72 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,9 +7,16 @@ edition = "2021" [dependencies] anyhow = "1.0.71" +async-openai = "0.10.3" clap = { version = "4.2.7", features = ["derive"] } clap-verbosity-flag = "2.0.1" +cpal = "0.15.2" +ctrlc = "3.2.5" +either = "1.8.1" +hound = "3.5.0" human-panic = "1.1.4" log = "0.4.17" pretty_env_logger = "0.4.0" rustyline = "11.0.0" +tempfile = "3.5.0" +tokio = { version = "1.28.1", features = ["rt-multi-thread"] } diff --git a/src/main.rs b/src/main.rs index 8ad6f1b..b660c74 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,24 @@ #![forbid(unsafe_code)] -use std::io; +use std::env; +use std::fs::File; +use std::io::BufWriter; use std::io::Write; +use std::io::{self}; +use std::iter::once; +use std::sync::mpsc::channel; +use std::sync::Arc; +use std::sync::Mutex; +use async_openai::types::CreateTranscriptionRequestArgs; +use async_openai::Client; use clap::Parser; +use cpal::traits::DeviceTrait; +use cpal::traits::HostTrait; +use cpal::traits::StreamTrait; +use cpal::FromSample; +use cpal::Sample; +use either::Either; use rustyline::error::ReadlineError; use rustyline::history::History; use rustyline::Cmd; @@ -20,7 +35,8 @@ struct Cli { verbosity: clap_verbosity_flag::Verbosity, } -fn main() -> anyhow::Result<()> { +#[tokio::main] +async fn main() -> anyhow::Result<()> { human_panic::setup_panic!(); let cli = Cli::parse(); @@ -29,21 +45,130 @@ fn main() -> anyhow::Result<()> { .init(); log::debug!("{cli:#?}"); + if let Some(line) = get_line_text()? { + process_line(&line)?; + } + + Ok(()) +} + +fn get_line_text() -> anyhow::Result> { let mut editor = Editor::with_config(Config::builder().auto_add_history(true).build())?; editor.set_helper(Some(())); editor.bind_sequence(KeyEvent::alt('\r'), Cmd::Newline); loop { match read_line(&mut editor) { - Ok(line) => process_line(&line)?, - Err(ReadlineError::Interrupted | ReadlineError::Eof) => break, + Ok(line) => return Ok(Some(line)), + Err(ReadlineError::Interrupted | ReadlineError::Eof) => return Ok(None), err @ Err(_) => { err?; } } } +} - Ok(()) +async fn get_line_audio() -> anyhow::Result> { + let host = cpal::default_host(); + + let devices = host.input_devices()?; + let devices = if let Some(device) = host.default_input_device() { + Either::Left(once(device).chain(devices)) + } else { + Either::Right(devices) + }; + + let mut devices_configs = devices.filter_map(|device| { + device + .default_input_config() + .ok() + .map(|config| (device, config)) + }); + + let Some((device, config)) = devices_configs.next() else { + anyhow::bail!("Failed to get default input config"); + }; + + let path = tempfile::Builder::new() + .prefix("murmur") + .suffix(".wav") + .tempfile()? + .into_temp_path(); + log::debug!("{path:#?}"); + let spec = wav_spec_from_config(&config)?; + let writer = hound::WavWriter::create(&path, spec)?; + let writer = Arc::new(Mutex::new(Some(writer))); + + let writer_2 = writer.clone(); + + let err_fn = move |err| { + log::error!("an error occurred on stream: {err}"); + }; + + let stream = match config.sample_format() { + cpal::SampleFormat::I8 => { + device.build_input_stream( + &config.into(), + move |data, _: &_| write_input_data::(data, &writer_2), + err_fn, + None, + )? + } + cpal::SampleFormat::I16 => { + device.build_input_stream( + &config.into(), + move |data, _: &_| write_input_data::(data, &writer_2), + err_fn, + None, + )? + } + cpal::SampleFormat::I32 => { + device.build_input_stream( + &config.into(), + move |data, _: &_| write_input_data::(data, &writer_2), + err_fn, + None, + )? + } + cpal::SampleFormat::F32 => { + device.build_input_stream( + &config.into(), + move |data, _: &_| write_input_data::(data, &writer_2), + err_fn, + None, + )? + } + sample_format => { + return Err(anyhow::Error::msg(format!( + "Unsupported sample format '{sample_format}'" + ))) + } + }; + + stream.play()?; + let (tx, rx) = channel(); + ctrlc::set_handler(move || tx.send(()).expect("should send signal on channel"))?; + + rx.recv()?; + drop(stream); + writer.lock().unwrap().take().unwrap().finalize()?; + + let response = Client::default() + .with_api_key(env::var("OPENAI_API_KEY")?) + .audio() + .transcribe( + CreateTranscriptionRequestArgs::default() + .model("whisper-1") + .file(&path) + .build()?, + ) + .await?; + + log::debug!("{response:#?}"); + + path.close()?; + Ok(Some(response.text)) + // TODO: send none if empty } #[inline] @@ -66,6 +191,41 @@ fn process_line(line: &str) -> io::Result<()> { Ok(()) } +fn sample_format(format: cpal::SampleFormat) -> hound::SampleFormat { + if format.is_float() { + hound::SampleFormat::Float + } else { + hound::SampleFormat::Int + } +} + +fn wav_spec_from_config(config: &cpal::SupportedStreamConfig) -> anyhow::Result { + let wav_spec = hound::WavSpec { + channels: config.channels() as _, + sample_rate: config.sample_rate().0 as _, + bits_per_sample: u16::try_from(config.sample_format().sample_size() * 8)?, + sample_format: sample_format(config.sample_format()), + }; + Ok(wav_spec) +} + +type WavWriterHandle = Arc>>>>; + +fn write_input_data(input: &[T], writer: &WavWriterHandle) +where + T: Sample, + U: Sample + hound::Sample + FromSample, +{ + if let Ok(mut guard) = writer.try_lock() { + if let Some(writer) = guard.as_mut() { + for &sample in input.iter() { + let sample: U = U::from_sample(sample); + writer.write_sample(sample).ok(); + } + } + } +} + #[cfg(test)] mod tests { use clap::CommandFactory; From 5c507848651e05aead0b9d113c00ff0a78b1bff7 Mon Sep 17 00:00:00 2001 From: "Felipe S. S. Schneider" Date: Sun, 14 May 2023 13:10:15 -0300 Subject: [PATCH 8/9] feat: actually allow the use of the recoding --- src/main.rs | 28 +++++++++++++++++++++++----- 1 file changed, 23 insertions(+), 5 deletions(-) diff --git a/src/main.rs b/src/main.rs index b660c74..e954500 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,9 +2,9 @@ use std::env; use std::fs::File; +use std::io; use std::io::BufWriter; use std::io::Write; -use std::io::{self}; use std::iter::once; use std::sync::mpsc::channel; use std::sync::Arc; @@ -31,6 +31,9 @@ use rustyline::KeyEvent; #[command(author, version, about)] #[command(propagate_version = true)] struct Cli { + #[arg(short, long)] + record: bool, + #[clap(flatten)] verbosity: clap_verbosity_flag::Verbosity, } @@ -45,14 +48,20 @@ async fn main() -> anyhow::Result<()> { .init(); log::debug!("{cli:#?}"); - if let Some(line) = get_line_text()? { + let line = if cli.record { + get_line_audio().await? + } else { + get_line_text().await? + }; + + if let Some(line) = line { process_line(&line)?; } Ok(()) } -fn get_line_text() -> anyhow::Result> { +async fn get_line_text() -> anyhow::Result> { let mut editor = Editor::with_config(Config::builder().auto_add_history(true).build())?; editor.set_helper(Some(())); editor.bind_sequence(KeyEvent::alt('\r'), Cmd::Newline); @@ -69,6 +78,10 @@ fn get_line_text() -> anyhow::Result> { } async fn get_line_audio() -> anyhow::Result> { + let mut stdout = io::stdout().lock(); + write!(stdout, "🔴 (Ctrl-C to stop recording)")?; + stdout.flush()?; + let host = cpal::default_host(); let devices = host.input_devices()?; @@ -167,8 +180,13 @@ async fn get_line_audio() -> anyhow::Result> { log::debug!("{response:#?}"); path.close()?; - Ok(Some(response.text)) - // TODO: send none if empty + writeln!(stdout)?; + + if response.text.trim().is_empty() { + Ok(None) + } else { + Ok(Some(response.text)) + } } #[inline] From 0c1059311237273cb7303cc59ad303c17a16bfd4 Mon Sep 17 00:00:00 2001 From: "Felipe S. S. Schneider" Date: Sun, 14 May 2023 13:35:46 -0300 Subject: [PATCH 9/9] feat: merge answer into the interface --- Cargo.toml | 6 +- src/main.rs | 164 ++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 164 insertions(+), 6 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 5f38c72..348f006 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,10 +13,14 @@ clap-verbosity-flag = "2.0.1" cpal = "0.15.2" ctrlc = "3.2.5" either = "1.8.1" +futures = "0.3.28" hound = "3.5.0" human-panic = "1.1.4" log = "0.4.17" pretty_env_logger = "0.4.0" rustyline = "11.0.0" +serde = "1.0.163" +serde_yaml = "0.9.21" tempfile = "3.5.0" -tokio = { version = "1.28.1", features = ["rt-multi-thread"] } +thiserror = "1.0.40" +tokio = { version = "1.28.1", features = ["rt-multi-thread", "io-std"] } diff --git a/src/main.rs b/src/main.rs index e954500..8d8afef 100644 --- a/src/main.rs +++ b/src/main.rs @@ -4,13 +4,18 @@ use std::env; use std::fs::File; use std::io; use std::io::BufWriter; +use std::io::Read; use std::io::Write; use std::iter::once; use std::sync::mpsc::channel; use std::sync::Arc; use std::sync::Mutex; +use async_openai::error::OpenAIError; +use async_openai::types::ChatCompletionRequestMessage; +use async_openai::types::CreateChatCompletionRequestArgs; use async_openai::types::CreateTranscriptionRequestArgs; +use async_openai::types::Role; use async_openai::Client; use clap::Parser; use cpal::traits::DeviceTrait; @@ -19,6 +24,7 @@ use cpal::traits::StreamTrait; use cpal::FromSample; use cpal::Sample; use either::Either; +use futures::StreamExt; use rustyline::error::ReadlineError; use rustyline::history::History; use rustyline::Cmd; @@ -26,11 +32,19 @@ use rustyline::Config; use rustyline::Editor; use rustyline::Helper; use rustyline::KeyEvent; +use serde::Deserialize; +use serde::Serialize; +use thiserror::Error; +use tokio::io::AsyncWrite; +use tokio::io::AsyncWriteExt; #[derive(Debug, Parser)] #[command(author, version, about)] #[command(propagate_version = true)] struct Cli { + #[arg(value_parser = parse_conversation)] + conversation: Option, + #[arg(short, long)] record: bool, @@ -38,6 +52,14 @@ struct Cli { verbosity: clap_verbosity_flag::Verbosity, } +#[derive(Debug, Error)] +enum CliError { + #[error("could not perform a serialization or deserialization operation: {0}")] + Yaml(#[from] serde_yaml::Error), + #[error("could not perform an input or output operation: {0}")] + Io(#[from] io::Error), +} + #[tokio::main] async fn main() -> anyhow::Result<()> { human_panic::setup_panic!(); @@ -55,12 +77,142 @@ async fn main() -> anyhow::Result<()> { }; if let Some(line) = line { - process_line(&line)?; + let conversation = cli.conversation.unwrap_or_default(); + process_line(conversation, &line).await?; } Ok(()) } +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct Conversation { + #[serde(default, skip_serializing_if = "Vec::is_empty")] + messages: Vec, +} + +impl Conversation { + #[inline] + fn push(&mut self, message: Message) { + self.messages.push(message); + } + + #[inline] + fn from_reader(reader: R) -> Result + where + R: Read, + { + serde_yaml::from_reader(reader) + } +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +struct Message { + #[serde(default, skip_serializing_if = "is_user")] + role: Role, + #[serde(default, skip_serializing_if = "String::is_empty")] + content: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + name: Option, +} + +impl Message { + #[inline] + fn from_user(content: C) -> Self + where + C: Into, + { + Self { + role: Role::User, + content: content.into(), + name: None, + } + } +} + +impl From for ChatCompletionRequestMessage { + #[inline] + fn from(message: Message) -> Self { + Self { + role: message.role, + content: message.content, + name: message.name, + } + } +} + +#[derive(Debug, Default, Serialize, Deserialize)] +struct Bot {} + +#[derive(Debug, Error)] +enum BotError { + #[error("could not obtain environment variable: {0}")] + Var(#[from] env::VarError), + #[error("could not exchange data with OpenAI: {0}")] + OpenAI(#[from] OpenAIError), + #[error("could not perform an input or output operation: {0}")] + Io(#[from] io::Error), +} + +impl Bot { + #[inline] + async fn reply_to_writer( + &self, + conversation: &Conversation, + mut writer: W, + ) -> Result<(), BotError> + where + W: AsyncWrite + Send + Unpin, + { + let mut stream = Client::default() + .with_api_key(env::var("OPENAI_API_KEY")?) + .chat() + .create_stream({ + CreateChatCompletionRequestArgs::default() + .model("gpt-3.5-turbo") + .temperature(0.0) + .messages( + conversation + .messages + .iter() + .cloned() + .map(Into::into) + .collect::>(), + ) + .build()? + }) + .await?; + + while let Some(response) = stream.next().await { + for content in response? + .choices + .into_iter() + .filter_map(|choice| choice.delta.content) + { + writer.write_all(content.as_bytes()).await?; + } + + writer.flush().await?; + } + + Ok(()) + } +} + +#[inline] +fn parse_conversation(path: &str) -> Result { + let file = File::open(path)?; + let conversation = Conversation::from_reader(file)?; + Ok(conversation) +} + +#[inline] +const fn is_user(role: &Role) -> bool { + match role { + Role::User => true, + Role::System | Role::Assistant => false, + } +} + async fn get_line_text() -> anyhow::Result> { let mut editor = Editor::with_config(Config::builder().auto_add_history(true).build())?; editor.set_helper(Some(())); @@ -202,10 +354,12 @@ fn read_line(editor: &mut Editor) -> rustyline::Res } #[inline] -fn process_line(line: &str) -> io::Result<()> { - let mut stdout = io::stdout().lock(); - writeln!(stdout, "GOT: {line}")?; - stdout.flush()?; +async fn process_line(mut conversation: Conversation, line: &str) -> anyhow::Result<()> { + conversation.push(Message::from_user(line)); + + Bot::default() + .reply_to_writer(&conversation, tokio::io::stdout()) + .await?; Ok(()) }