open System.IO
open System
let tokenize (delims:char seq) str =
let delims = delims |> Seq.map int |> set
seq {
use strr = new StringReader(str)
let rec tokens (acc,s:StringReader) =
seq {
match s.Peek() with
| -1 -> ()
| d when delims.Contains d ->
let _ = s.Read()
if List.length acc > 0 then
let str = new String(List.rev acc |> List.toArray)
yield str
yield! tokens([],s)
else
yield! tokens ([],s)
| c ->
let _ = s.Read()
yield! tokens (char c::acc,s)
}
yield! tokens ([],strr)
}
(*
let str = "ab c 12 d, (n)"
let delims = [|' '; '('; ')'; ',' |]
tokenize delims str |> Seq.iter (printfn "%A")
*)
