Completely migrate to prism

Gemfile

@@ -28,6 +28,8 @@ if ENV['PRISM_VERSION'] == 'latest'
   gem "prism", github: "ruby/prism"
 elsif ENV['PRISM_VERSION']
   gem "prism", ENV['PRISM_VERSION']
+else
+  gem "prism", "!= 1.8.0"
 end
 
 if RUBY_VERSION >= "3.0.0" && !is_truffleruby

lib/irb.rb

@@ -5,7 +5,7 @@
 #       by Keiju ISHITSUKA([email protected])
 #
 
-require "ripper"
+require "prism"
 require "reline"
 
 require_relative "irb/init"
@@ -251,11 +251,10 @@ def read_input_nomultiline(prompt)
         code << line
         return code if command?(code)
 
-        tokens, opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables)
+        continue, opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables)
         return code if terminated
 
         line_offset += 1
-        continue = @scanner.should_continue?(tokens)
         prompt = generate_prompt(opens, continue, line_offset)
       end
     end
@@ -316,23 +315,25 @@ def configure_io
           else
             next true if command?(code)
 
-            _tokens, _opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables)
+            _continue, _opens, terminated = @scanner.check_code_state(code, local_variables: @context.local_variables)
             terminated
           end
         end
       end
       if @context.io.respond_to?(:dynamic_prompt)
         @context.io.dynamic_prompt do |lines|
-          tokens = RubyLex.ripper_lex_without_warning(lines.map{ |l| l + "\n" }.join, local_variables: @context.local_variables)
-          line_results = IRB::NestingParser.parse_by_line(tokens)
+          code = lines.map{ |l| l + "\n" }.join
+          parse_lex_result = Prism.parse_lex(code, scopes: [@context.local_variables])
+          line_results = IRB::NestingParser.parse_by_line(parse_lex_result)
+
+          tokens = parse_lex_result.value[1].map(&:first)
+          tokens_by_line = tokens.group_by {|t| t.location.start_line - 1 }
+
           tokens_until_line = []
-          line_results.map.with_index do |(line_tokens, _prev_opens, next_opens, _min_depth), line_num_offset|
-            line_tokens.each do |token, _s|
-              # Avoid appending duplicated token. Tokens that include "n" like multiline
-              # tstring_content can exist in multiple lines.
-              tokens_until_line << token if token != tokens_until_line.last
-            end
-            continue = @scanner.should_continue?(tokens_until_line)
+          line_results.map.with_index do |(_prev_opens, next_opens, _min_depth), line_num_offset|
+            line = lines[line_num_offset]
+            tokens_until_line.concat(tokens_by_line[line_num_offset] || [])
+            continue = @scanner.should_continue?(tokens_until_line, line, line_num_offset + 1)
             generate_prompt(next_opens, continue, line_num_offset)
           end
         end
@@ -344,8 +345,8 @@ def configure_io
           next nil if !is_newline && lines[line_index]&.byteslice(0, byte_pointer)&.match?(/\A\s*\z/)
 
           code = lines[0..line_index].map { |l| "#{l}\n" }.join
-          tokens = RubyLex.ripper_lex_without_warning(code, local_variables: @context.local_variables)
-          @scanner.process_indent_level(tokens, lines, line_index, is_newline)
+          parse_lex_result = Prism.parse_lex(code, scopes: [@context.local_variables])
+          @scanner.process_indent_level(parse_lex_result, lines, line_index, is_newline)
         end
       end
     end

lib/irb/command/internal_helpers.rb

@@ -1,15 +1,18 @@
 # frozen_string_literal: true
 
+require 'prism'
+
 module IRB
   module Command
     # Internal use only, for default command's backward compatibility.
     module RubyArgsExtractor # :nodoc:
       def unwrap_string_literal(str)
         return if str.empty?
 
-        sexp = Ripper.sexp(str)
-        if sexp && sexp.size == 2 && sexp.last&.first&.first == :string_literal
-          @irb_context.workspace.binding.eval(str).to_s
+        result = Prism.parse(str)
+        body = result.value.statements.body
+        if result.success? && body.size == 1 && body.first.is_a?(Prism::StringNode)
+          body.first.unescaped
         else
           str
         end

lib/irb/completion.rb

@@ -166,22 +166,13 @@ def complete_require_path(target, preposing, postposing)
       else
         return nil # It's not String literal
       end
-      tokens = RubyLex.ripper_lex_without_warning(preposing.rstrip)
-      tok = nil
-      tokens.reverse_each do |t|
-        unless [:on_lparen, :on_sp, :on_ignored_sp, :on_nl, :on_ignored_nl, :on_comment].include?(t.event)
-          tok = t
-          break
-        end
-      end
-      return unless tok&.event == :on_ident && tok.state == Ripper::EXPR_CMDARG
 
-      case tok.tok
-      when 'require'
+      case preposing
+      when /(^|[^\w])require\(? *\z/
         retrieve_files_to_require_from_load_path.filter_map { |path|
           quote + path if path.start_with?(actual_target)
         }
-      when 'require_relative'
+      when /(^|[^\w])require_relative\(? *\z/
         retrieve_files_to_require_relative_from_current_dir.filter_map { |path|
           quote + path if path.start_with?(actual_target)
         }