Class RDoc::RubyParser
In: lib/rdoc/parsers/parse_rb.rb
Parent: Object

Methods

Included Modules

RubyToken TokenStream

Constants

NORMAL = "::"
SINGLE = "<<"

Public Class methods

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1387
1387:     def initialize(top_level, file_name, content, options, stats)
1388:       @options = options
1389:       @stats   = stats
1390:       @size = 0
1391:       @token_listeners = nil
1392:       @input_file_name = file_name
1393:       @scanner = RubyLex.new(content)
1394:       @scanner.exception_on_syntax_error = false
1395:       @top_level = top_level
1396:       @progress = $stderr unless options.quiet
1397:     end

Public Instance methods

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1399
1399:     def scan
1400:       @tokens = []
1401:       @unget_read = []
1402:       @read = []
1403:       catch(:eof) do
1404:         catch(:enddoc) do
1405:           begin
1406:             parse_toplevel_statements(@top_level)
1407:           rescue Exception => e
1408:             $stderr.puts "\n\n"
1409:             $stderr.puts "RDoc failure in #@input_file_name at or around " +
1410:                          "line #{@scanner.line_no} column #{@scanner.char_no}"
1411:             $stderr.puts 
1412:             $stderr.puts "Before reporting this, could you check that the file"
1413:             $stderr.puts "you're documenting compiles cleanly--RDoc is not a"
1414:             $stderr.puts "full Ruby parser, and gets confused easily if fed"
1415:             $stderr.puts "invalid programs."
1416:             $stderr.puts
1417:             $stderr.puts "The internal error was:\n\n"
1418:             
1419:             e.set_backtrace(e.backtrace[0,4])
1420:             raise
1421:           end
1422:         end
1423:       end
1424:       @top_level
1425:     end

Private Instance methods

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1456
1456:     def add_token_listener(obj)
1457:       @token_listeners ||= []
1458:       @token_listeners << obj
1459:     end

Look for the first comment in a file that isn‘t a shebang line.

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1542
1542:     def collect_first_comment
1543:       skip_tkspace
1544:       res = ''
1545:       first_line = true
1546: 
1547:       tk = get_tk
1548:       while tk.kind_of?(TkCOMMENT)
1549:         if first_line && /\A#!/ =~ tk.text
1550:           skip_tkspace
1551:           tk = get_tk
1552:         elsif first_line && /\A#\s*-\*-/ =~ tk.text
1553:           first_line = false
1554:           skip_tkspace
1555:           tk = get_tk
1556:         else
1557:           first_line = false
1558:           res << tk.text << "\n"
1559:           tk = get_tk
1560:           if tk.kind_of? TkNL
1561:             skip_tkspace(false)
1562:             tk = get_tk
1563:           end
1564:         end
1565:       end
1566:       unget_tk(tk)
1567:       res
1568:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1443
1443:     def error(msg)
1444:       msg = make_message msg
1445:       $stderr.puts msg
1446:       exit(1)
1447:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2445
2445:     def get_bool
2446:       skip_tkspace
2447:       tk = get_tk
2448:       case tk
2449:       when TkTRUE
2450:         true
2451:       when TkFALSE, TkNIL
2452:         false
2453:       else
2454:         unget_tk tk
2455:         true
2456:       end
2457:     end
Look for the name of a class of module (optionally with a leading :or
with :separated named) and return the ultimate name and container

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1801
1801:     def get_class_or_module(container)
1802:       skip_tkspace
1803:       name_t = get_tk
1804: 
1805:       # class ::A -> A is in the top level
1806:       if name_t.kind_of?(TkCOLON2)
1807:         name_t = get_tk
1808:         container = @top_level
1809:       end
1810: 
1811:       skip_tkspace(false)
1812: 
1813:       while peek_tk.kind_of?(TkCOLON2)
1814:         prev_container = container
1815:         container = container.find_module_named(name_t.name)
1816:         if !container
1817: #          warn("Couldn't find module #{name_t.name}")
1818:           container = prev_container.add_module(NormalModule, name_t.name)
1819:         end
1820:         get_tk
1821:         name_t = get_tk
1822:       end
1823:       skip_tkspace(false)
1824:       return [container, name_t]
1825:     end

Return a superclass, which can be either a constant of an expression

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2130
2130:     def get_class_specification
2131:       tk = get_tk
2132:       return "self" if tk.kind_of?(TkSELF)
2133:         
2134:       res = ""
2135:       while tk.kind_of?(TkCOLON2) ||
2136:           tk.kind_of?(TkCOLON3)   ||
2137:           tk.kind_of?(TkCONSTANT)   
2138:         
2139:         res += tk.text
2140:         tk = get_tk
2141:       end
2142: 
2143:       unget_tk(tk)
2144:       skip_tkspace(false)
2145: 
2146:       get_tkread # empty out read buffer
2147: 
2148:       tk = get_tk
2149: 
2150:       case tk
2151:       when TkNL, TkCOMMENT, TkSEMICOLON
2152:         unget_tk(tk)
2153:         return res
2154:       end
2155: 
2156:       res += parse_call_parameters(tk)
2157:       res
2158:     end

Parse a constant, which might be qualified by one or more class or module names

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2202
2202:     def get_constant
2203:       res = ""
2204:       skip_tkspace(false)
2205:       tk = get_tk
2206: 
2207:       while tk.kind_of?(TkCOLON2) ||
2208:           tk.kind_of?(TkCOLON3)   ||
2209:           tk.kind_of?(TkCONSTANT)          
2210:         
2211:         res += tk.text
2212:         tk = get_tk
2213:       end
2214: 
2215: #      if res.empty?
2216: #        warn("Unexpected token #{tk} in constant")
2217: #      end 
2218:       unget_tk(tk)
2219:       res
2220:     end

Get a constant that may be surrounded by parens

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2224
2224:     def get_constant_with_optional_parens
2225:       skip_tkspace(false)
2226:       nest = 0
2227:       while (tk = peek_tk).kind_of?(TkLPAREN)  || tk.kind_of?(TkfLPAREN)
2228:         get_tk
2229:         skip_tkspace(true)
2230:         nest += 1
2231:       end
2232: 
2233:       name = get_constant
2234: 
2235:       while nest > 0
2236:         skip_tkspace(true)
2237:         tk = get_tk
2238:         nest -= 1 if tk.kind_of?(TkRPAREN)
2239:       end
2240:       name
2241:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2359
2359:     def get_symbol_or_name
2360:       tk = get_tk
2361:       case tk
2362:       when  TkSYMBOL
2363:         tk.text.sub(/^:/, '')
2364:       when TkId, TkOp
2365:         tk.name
2366:       when TkSTRING
2367:         tk.text
2368:       else
2369:         raise "Name or symbol expected (got #{tk})"
2370:       end
2371:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1465
1465:     def get_tk
1466:       tk = nil
1467:       if @tokens.empty?
1468:         tk = @scanner.token
1469:         @read.push @scanner.get_read
1470:         puts "get_tk1 => #{tk.inspect}" if $TOKEN_DEBUG
1471:       else
1472:         @read.push @unget_read.shift
1473:         tk = @tokens.shift
1474:         puts "get_tk2 => #{tk.inspect}" if $TOKEN_DEBUG
1475:       end
1476: 
1477:       if tk.kind_of?(TkSYMBEG)
1478:         set_token_position(tk.line_no, tk.char_no)
1479:         tk1 = get_tk
1480:         if tk1.kind_of?(TkId) || tk1.kind_of?(TkOp)
1481:           tk = Token(TkSYMBOL).set_text(":" + tk1.name)
1482:           # remove the identifier we just read (we're about to
1483:           # replace it with a symbol)
1484:           @token_listeners.each do |obj|
1485:             obj.pop_token
1486:           end if @token_listeners
1487:         else
1488:           warn("':' not followed by identifier or operator")
1489:           tk = tk1
1490:         end
1491:       end
1492: 
1493:       # inform any listeners of our shiny new token
1494:       @token_listeners.each do |obj|
1495:         obj.add_token(tk)
1496:       end if @token_listeners
1497: 
1498:       tk
1499:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1526
1526:     def get_tkread
1527:       read = @read.join("")
1528:       @read = []
1529:       read
1530:     end

Look for directives in a normal comment block:

  #--       - don't display comment from this point forward

This routine modifies it‘s parameter

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2309
2309:     def look_for_directives_in(context, comment)
2310: 
2311:       preprocess = SM::PreProcess.new(@input_file_name,
2312:                                       @options.rdoc_include)
2313: 
2314:       preprocess.handle(comment) do |directive, param|
2315:         case directive
2316:         when "stopdoc"
2317:           context.stop_doc
2318:           ""
2319:         when "startdoc"
2320:           context.start_doc
2321:           context.force_documentation = true
2322:           ""
2323: 
2324:         when "enddoc"
2325:           #context.done_documenting = true
2326:           #""
2327:           throw :enddoc
2328: 
2329:         when "main"
2330:           options = Options.instance
2331:           options.main_page = param
2332:           ""
2333: 
2334:         when "title"
2335:           options = Options.instance
2336:           options.title = param
2337:           ""
2338: 
2339:         when "section"
2340:           context.set_current_section(param, comment)
2341:           comment.replace("") # 1.8 doesn't support #clear
2342:           break 
2343:         else
2344:           warn "Unrecognized directive '#{directive}'"
2345:           break
2346:         end
2347:       end
2348: 
2349:       remove_private_comments(comment)
2350:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1429
1429:     def make_message(msg)
1430:       prefix = "\n" + @input_file_name + ":"
1431:       if @scanner
1432:         prefix << "#{@scanner.line_no}:#{@scanner.char_no}: "
1433:       end
1434:       return prefix + msg
1435:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2373
2373:     def parse_alias(context, single, tk, comment)
2374:       skip_tkspace
2375:       if (peek_tk.kind_of? TkLPAREN)
2376:         get_tk
2377:         skip_tkspace
2378:       end
2379:       new_name = get_symbol_or_name
2380:       @scanner.instance_eval{@lex_state = EXPR_FNAME}
2381:       skip_tkspace
2382:       if (peek_tk.kind_of? TkCOMMA)
2383:         get_tk
2384:         skip_tkspace
2385:       end
2386:       old_name = get_symbol_or_name
2387: 
2388:       al = Alias.new(get_tkread, old_name, new_name, comment)
2389:       read_documentation_modifiers(al, ATTR_MODIFIERS)
2390:       if al.document_self
2391:         context.add_alias(al)
2392:       end
2393:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2459
2459:     def parse_attr(context, single, tk, comment)
2460:       args = parse_symbol_arg(1)
2461:       if args.size > 0
2462:         name = args[0]
2463:         rw = "R"
2464:         skip_tkspace(false)
2465:         tk = get_tk
2466:         if tk.kind_of? TkCOMMA
2467:           rw = "RW" if get_bool
2468:         else
2469:           unget_tk tk
2470:         end
2471:         att = Attr.new(get_tkread, name, rw, comment)
2472:         read_documentation_modifiers(att, ATTR_MODIFIERS)
2473:         if att.document_self
2474:           context.add_attribute(att)
2475:         end
2476:       else
2477:         warn("'attr' ignored - looks like a variable")
2478:       end    
2479: 
2480:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2513
2513:     def parse_attr_accessor(context, single, tk, comment)
2514:       args = parse_symbol_arg
2515:       read = get_tkread
2516:       rw = "?"
2517: 
2518:       # If nodoc is given, don't document any of them
2519: 
2520:       tmp = CodeObject.new
2521:       read_documentation_modifiers(tmp, ATTR_MODIFIERS)
2522:       return unless tmp.document_self
2523: 
2524:       case tk.name
2525:       when "attr_reader"   then rw = "R"
2526:       when "attr_writer"   then rw = "W"
2527:       when "attr_accessor" then rw = "RW"
2528:       else
2529:         rw = @options.extra_accessor_flags[tk.name]
2530:       end
2531:       
2532:       for name in args
2533:         att = Attr.new(get_tkread, name, rw, comment)
2534:         context.add_attribute(att)
2535:       end    
2536:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2160
2160:     def parse_call_parameters(tk)
2161: 
2162:       end_token = case tk
2163:                   when TkLPAREN, TkfLPAREN
2164:                     TkRPAREN
2165:                   when TkRPAREN
2166:                     return ""
2167:                   else
2168:                     TkNL
2169:                   end
2170:       nest = 0
2171: 
2172:       loop do
2173:         puts("Call param: #{tk}, #{@scanner.continue} " +
2174:           "#{@scanner.lex_state} #{nest}") if $DEBUG
2175:         case tk
2176:         when TkSEMICOLON
2177:           break
2178:         when TkLPAREN, TkfLPAREN
2179:           nest += 1
2180:         when end_token
2181:           if end_token == TkRPAREN
2182:             nest -= 1
2183:             break if @scanner.lex_state == EXPR_END and nest <= 0
2184:           else
2185:             break unless @scanner.continue
2186:           end
2187:         when TkCOMMENT
2188:           unget_tk(tk)
2189:           break
2190:         end
2191:         tk = get_tk
2192:       end
2193:       res = get_tkread.tr("\n", " ").strip
2194:       res = "" if res == ";"
2195:       res
2196:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1733
1733:     def parse_class(container, single, tk, comment, &block)
1734:       progress("c")
1735: 
1736:       @stats.num_classes += 1
1737: 
1738:       container, name_t = get_class_or_module(container)
1739: 
1740:       case name_t
1741:       when TkCONSTANT
1742:         name = name_t.name
1743:         superclass = "Object"
1744: 
1745:         if peek_tk.kind_of?(TkLT)
1746:           get_tk
1747:           skip_tkspace(true)
1748:           superclass = get_class_specification
1749:           superclass = "<unknown>" if superclass.empty?
1750:         end
1751: 
1752:         if single == SINGLE
1753:           cls_type = SingleClass
1754:         else
1755:           cls_type = NormalClass
1756:         end
1757: 
1758:         cls = container.add_class(cls_type, name, superclass)
1759:         read_documentation_modifiers(cls, CLASS_MODIFIERS)
1760:         cls.record_location(@top_level)
1761:         parse_statements(cls)
1762:         cls.comment = comment
1763: 
1764:       when TkLSHFT
1765:         case name = get_class_specification
1766:         when "self", container.name
1767:           parse_statements(container, SINGLE, &block)
1768:         else
1769:           other = TopLevel.find_class_named(name)
1770:           unless other
1771: #            other = @top_level.add_class(NormalClass, name, nil)
1772: #            other.record_location(@top_level)
1773: #            other.comment = comment
1774:             other = NormalClass.new("Dummy", nil)
1775:           end
1776:           read_documentation_modifiers(other, CLASS_MODIFIERS)
1777:           parse_statements(other, SINGLE, &block)
1778:         end
1779: 
1780:       else
1781:         warn("Expected class name or '<<'. Got #{name_t.class}: #{name_t.text.inspect}")
1782:       end
1783:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1827
1827:     def parse_constant(container, single, tk, comment)
1828:       name = tk.name
1829:       skip_tkspace(false)
1830:       eq_tk = get_tk
1831: 
1832:       unless eq_tk.kind_of?(TkASSIGN)
1833:         unget_tk(eq_tk)
1834:         return
1835:       end
1836: 
1837: 
1838:       nest = 0
1839:       get_tkread
1840: 
1841:       tk = get_tk
1842:       if tk.kind_of? TkGT
1843:         unget_tk(tk)
1844:         unget_tk(eq_tk)
1845:         return
1846:       end
1847: 
1848:       loop do
1849:         puts("Param: #{tk}, #{@scanner.continue} " +
1850:           "#{@scanner.lex_state} #{nest}")  if $DEBUG
1851: 
1852:         case tk
1853:         when TkSEMICOLON
1854:           break
1855:         when TkLPAREN, TkfLPAREN
1856:           nest += 1
1857:         when TkRPAREN
1858:           nest -= 1
1859:         when TkCOMMENT
1860:           if nest <= 0 && @scanner.lex_state == EXPR_END
1861:             unget_tk(tk)
1862:             break
1863:           end
1864:         when TkNL
1865:           if (@scanner.lex_state == EXPR_END and nest <= 0) || !@scanner.continue
1866:             unget_tk(tk)
1867:             break
1868:           end
1869:         end
1870:         tk = get_tk
1871:       end
1872: 
1873:       res = get_tkread.tr("\n", " ").strip
1874:       res = "" if res == ";"
1875:       con = Constant.new(name, res, comment)
1876:       read_documentation_modifiers(con, CONSTANT_MODIFIERS)
1877:       if con.document_self
1878:         container.add_constant(con)
1879:       end
1880:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2433
2433:   def parse_include(context, comment)
2434:     loop do
2435:       skip_tkspace_comment
2436:       name = get_constant_with_optional_parens
2437:       unless name.empty?
2438:         context.add_include(Include.new(name, comment))
2439:       end
2440:       return unless peek_tk.kind_of?(TkCOMMA)
2441:       get_tk
2442:     end
2443:   end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1882
1882:     def parse_method(container, single, tk, comment)
1883:       progress(".")
1884:       @stats.num_methods += 1
1885:       line_no = tk.line_no
1886:       column  = tk.char_no
1887:       
1888:       start_collecting_tokens
1889:       add_token(tk)
1890:       add_token_listener(self)
1891:       
1892:       @scanner.instance_eval{@lex_state = EXPR_FNAME}
1893:       skip_tkspace(false)
1894:       name_t = get_tk
1895:       back_tk = skip_tkspace
1896:       meth = nil
1897:       added_container = false
1898: 
1899:       dot = get_tk
1900:       if dot.kind_of?(TkDOT) or dot.kind_of?(TkCOLON2)
1901:         @scanner.instance_eval{@lex_state = EXPR_FNAME}
1902:         skip_tkspace
1903:         name_t2 = get_tk
1904:         case name_t
1905:         when TkSELF
1906:           name = name_t2.name
1907:         when TkCONSTANT
1908:           name = name_t2.name
1909:           prev_container = container
1910:           container = container.find_module_named(name_t.name)
1911:           if !container
1912:             added_container = true
1913:             obj = name_t.name.split("::").inject(Object) do |state, item|
1914:               state.const_get(item)
1915:             end rescue nil
1916: 
1917:             type = obj.class == Class ? NormalClass : NormalModule
1918:             if not [Class, Module].include?(obj.class)
1919:               warn("Couldn't find #{name_t.name}. Assuming it's a module")
1920:             end
1921: 
1922:             if type == NormalClass then
1923:               container = prev_container.add_class(type, name_t.name, obj.superclass.name)
1924:             else
1925:               container = prev_container.add_module(type, name_t.name)
1926:             end
1927:           end
1928:         else
1929:           # warn("Unexpected token '#{name_t2.inspect}'")
1930:           # break
1931:           skip_method(container)
1932:           return
1933:         end
1934:         meth =  AnyMethod.new(get_tkread, name)
1935:         meth.singleton = true
1936:       else
1937:         unget_tk dot
1938:         back_tk.reverse_each do
1939:           |tk|
1940:           unget_tk tk
1941:         end
1942:         name = name_t.name
1943: 
1944:         meth =  AnyMethod.new(get_tkread, name)
1945:         meth.singleton = (single == SINGLE)
1946:       end
1947: 
1948:       remove_token_listener(self)
1949: 
1950:       meth.start_collecting_tokens
1951:       indent = TkSPACE.new(1,1)
1952:       indent.set_text(" " * column)
1953: 
1954:       meth.add_tokens([TkCOMMENT.new(line_no,
1955:                                      1,
1956:                                      "# File #{@top_level.file_absolute_name}, line #{line_no}"),
1957:                         NEWLINE_TOKEN,
1958:                         indent])
1959: 
1960:       meth.add_tokens(@token_stream)
1961: 
1962:       add_token_listener(meth)
1963: 
1964:       @scanner.instance_eval{@continue = false}
1965:       parse_method_parameters(meth)
1966: 
1967:       if meth.document_self
1968:         container.add_method(meth)
1969:       elsif added_container
1970:         container.document_self = false
1971:       end
1972: 
1973:       # Having now read the method parameters and documentation modifiers, we
1974:       # now know whether we have to rename #initialize to ::new
1975: 
1976:       if name == "initialize" && !meth.singleton
1977:         if meth.dont_rename_initialize
1978:           meth.visibility = :protected
1979:         else
1980:           meth.singleton = true
1981:           meth.name = "new"
1982:           meth.visibility = :public
1983:         end
1984:       end
1985:       
1986:       parse_statements(container, single, meth)
1987:       
1988:       remove_token_listener(meth)
1989: 
1990:       # Look for a 'call-seq' in the comment, and override the
1991:       # normal parameter stuff
1992: 
1993:       if comment.sub!(/:?call-seq:(.*?)^\s*\#?\s*$/m, '')
1994:         seq = $1
1995:         seq.gsub!(/^\s*\#\s*/, '')
1996:         meth.call_seq = seq
1997:       end
1998:       
1999:       meth.comment = comment
2000: 
2001:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2026
2026:     def parse_method_or_yield_parameters(method=nil, modifiers=METHOD_MODIFIERS)
2027:       skip_tkspace(false)
2028:       tk = get_tk
2029: 
2030:       # Little hack going on here. In the statement
2031:       #  f = 2*(1+yield)
2032:       # We see the RPAREN as the next token, so we need
2033:       # to exit early. This still won't catch all cases
2034:       # (such as "a = yield + 1"
2035:       end_token = case tk
2036:                   when TkLPAREN, TkfLPAREN
2037:                     TkRPAREN
2038:                   when TkRPAREN
2039:                     return ""
2040:                   else
2041:                     TkNL
2042:                   end
2043:       nest = 0
2044: 
2045:       loop do
2046:         puts("Param: #{tk.inspect}, #{@scanner.continue} " +
2047:           "#{@scanner.lex_state} #{nest}")  if $DEBUG
2048:         case tk
2049:         when TkSEMICOLON
2050:           break
2051:         when TkLBRACE
2052:           nest += 1
2053:         when TkRBRACE
2054:           # we might have a.each {|i| yield i }
2055:           unget_tk(tk) if nest.zero?
2056:           nest -= 1
2057:           break if nest <= 0
2058:         when TkLPAREN, TkfLPAREN
2059:           nest += 1
2060:         when end_token
2061:           if end_token == TkRPAREN
2062:             nest -= 1
2063:             break if @scanner.lex_state == EXPR_END and nest <= 0
2064:           else
2065:             break unless @scanner.continue
2066:           end
2067:         when method && method.block_params.nil? && TkCOMMENT
2068:           unget_tk(tk)
2069:           read_documentation_modifiers(method, modifiers)
2070:         end
2071:         tk = get_tk
2072:       end
2073:       res = get_tkread.tr("\n", " ").strip
2074:       res = "" if res == ";"
2075:       res
2076:     end

Capture the method‘s parameters. Along the way, look for a comment containing

   # yields: ....

and add this as the block_params for the method

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2016
2016:     def parse_method_parameters(method)
2017:       res = parse_method_or_yield_parameters(method)
2018:       res = "(" + res + ")" unless res[0] == ?(
2019:       method.params = res unless method.params
2020:       if method.block_params.nil?
2021:           skip_tkspace(false)
2022:           read_documentation_modifiers(method, METHOD_MODIFIERS)
2023:       end
2024:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1785
1785:     def parse_module(container, single, tk, comment)
1786:       progress("m")
1787:       @stats.num_modules += 1
1788:       container, name_t  = get_class_or_module(container)
1789: #      skip_tkspace
1790:       name = name_t.name
1791:       mod = container.add_module(NormalModule, name)
1792:       mod.record_location(@top_level)
1793:       read_documentation_modifiers(mod, CLASS_MODIFIERS)
1794:       parse_statements(mod)
1795:       mod.comment = comment
1796:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2407
2407:   def parse_require(context, comment)
2408:     skip_tkspace_comment
2409:     tk = get_tk
2410:     if tk.kind_of? TkLPAREN
2411:       skip_tkspace_comment
2412:       tk = get_tk
2413:     end
2414: 
2415:     name = nil
2416:     case tk
2417:     when TkSTRING
2418:       name = tk.text
2419: #    when TkCONSTANT, TkIDENTIFIER, TkIVAR, TkGVAR
2420: #      name = tk.name
2421:     when TkDSTRING
2422:       warn "Skipping require of dynamic string: #{tk.text}"
2423:  #   else
2424:  #     warn "'require' used as variable"
2425:     end
2426:     if name
2427:       context.add_require(Require.new(name, comment))
2428:     else
2429:       unget_tk(tk)
2430:     end
2431:   end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1577
1577:     def parse_statements(container, single=NORMAL, current_method=nil, comment='')
1578:       nest = 1
1579:       save_visibility = container.visibility
1580:       
1581: #      if container.kind_of?(TopLevel)
1582: #      else
1583: #        comment = ''
1584: #      end
1585: 
1586:       non_comment_seen = true
1587:       
1588:       while tk = get_tk
1589:         
1590:         keep_comment = false
1591:         
1592:         non_comment_seen = true unless tk.kind_of?(TkCOMMENT)
1593:         
1594:         case tk
1595: 
1596:         when TkNL
1597:           skip_tkspace(true)   # Skip blanks and newlines
1598:           tk = get_tk
1599:           if tk.kind_of?(TkCOMMENT)
1600:             if non_comment_seen
1601:               comment = ''
1602:               non_comment_seen = false
1603:             end
1604:             while tk.kind_of?(TkCOMMENT)
1605:               comment << tk.text << "\n"
1606:               tk = get_tk          # this is the newline 
1607:               skip_tkspace(false)  # leading spaces
1608:               tk = get_tk
1609:             end
1610:             unless comment.empty?
1611:               look_for_directives_in(container, comment) 
1612:               if container.done_documenting
1613:                 container.ongoing_visibility = save_visibility
1614: #                return
1615:               end
1616:             end
1617:             keep_comment = true
1618:           else
1619:             non_comment_seen = true
1620:           end
1621:           unget_tk(tk)
1622:           keep_comment = true
1623: 
1624: 
1625:         when TkCLASS
1626:           if container.document_children
1627:             parse_class(container, single, tk, comment)
1628:           else
1629:             nest += 1
1630:           end
1631: 
1632:         when TkMODULE
1633:           if container.document_children
1634:             parse_module(container, single, tk, comment)
1635:           else
1636:             nest += 1
1637:           end
1638: 
1639:         when TkDEF
1640:           if container.document_self
1641:             parse_method(container, single, tk, comment)
1642:           else
1643:             nest += 1
1644:           end
1645: 
1646:         when TkCONSTANT
1647:           if container.document_self
1648:             parse_constant(container, single, tk, comment)
1649:           end
1650: 
1651:         when TkALIAS
1652:           if container.document_self
1653:             parse_alias(container, single, tk, comment)
1654:           end
1655: 
1656:         when TkYIELD
1657:           if current_method.nil?
1658:             warn("Warning: yield outside of method") if container.document_self
1659:           else
1660:             parse_yield(container, single, tk, current_method)
1661:           end
1662: 
1663:           # Until and While can have a 'do', which shouldn't increas
1664:           # the nesting. We can't solve the general case, but we can
1665:           # handle most occurrences by ignoring a do at the end of a line
1666: 
1667:         when  TkUNTIL, TkWHILE
1668:           nest += 1
1669:           puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
1670:             "line #{tk.line_no}" if $DEBUG
1671:           skip_optional_do_after_expression
1672: 
1673:           # 'for' is trickier
1674:         when TkFOR
1675:           nest += 1
1676:           puts "FOUND #{tk.class} in #{container.name}, nest = #{nest}, " +
1677:             "line #{tk.line_no}" if $DEBUG
1678:           skip_for_variable
1679:           skip_optional_do_after_expression
1680: 
1681:         when TkCASE, TkDO, TkIF, TkUNLESS, TkBEGIN
1682:           nest += 1
1683:           puts "Found #{tk.class} in #{container.name}, nest = #{nest}, " +
1684:             "line #{tk.line_no}" if $DEBUG
1685: 
1686:         when TkIDENTIFIER
1687:           if nest == 1 and current_method.nil?
1688:             case tk.name
1689:             when "private", "protected", "public",
1690:                  "private_class_method", "public_class_method"
1691:               parse_visibility(container, single, tk)
1692:               keep_comment = true
1693:             when "attr"
1694:               parse_attr(container, single, tk, comment)
1695:             when /^attr_(reader|writer|accessor)$/, @options.extra_accessors
1696:               parse_attr_accessor(container, single, tk, comment)
1697:             when "alias_method"
1698:               if container.document_self
1699:                 parse_alias(container, single, tk, comment)
1700:               end
1701:             end
1702:           end
1703:           
1704:           case tk.name
1705:           when "require"
1706:             parse_require(container, comment)
1707:           when "include"
1708:             parse_include(container, comment)
1709:           end
1710: 
1711: 
1712:         when TkEND
1713:           nest -= 1
1714:           puts "Found 'end' in #{container.name}, nest = #{nest}, line #{tk.line_no}" if $DEBUG
1715:           puts "Method = #{current_method.name}" if $DEBUG and current_method
1716:           if nest == 0
1717:             read_documentation_modifiers(container, CLASS_MODIFIERS)
1718:             container.ongoing_visibility = save_visibility
1719:             return
1720:           end
1721: 
1722:         end
1723: 
1724:         comment = '' unless keep_comment
1725:         begin
1726:           get_tkread
1727:           skip_tkspace(false)
1728:         end while peek_tk == TkNL
1729: 
1730:       end
1731:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2546
2546:     def parse_symbol_arg(no = nil)
2547: 
2548:       args = []
2549:       skip_tkspace_comment
2550:       case tk = get_tk
2551:       when TkLPAREN
2552:         loop do
2553:           skip_tkspace_comment
2554:           if tk1 = parse_symbol_in_arg
2555:             args.push tk1
2556:             break if no and args.size >= no
2557:           end
2558:           
2559:           skip_tkspace_comment
2560:           case tk2 = get_tk
2561:           when TkRPAREN
2562:             break
2563:           when TkCOMMA
2564:           else
2565:            warn("unexpected token: '#{tk2.inspect}'") if $DEBUG
2566:             break
2567:           end
2568:         end
2569:       else
2570:         unget_tk tk
2571:         if tk = parse_symbol_in_arg
2572:           args.push tk
2573:           return args if no and args.size >= no
2574:         end
2575: 
2576:         loop do
2577: #         skip_tkspace_comment(false)
2578:           skip_tkspace(false)
2579: 
2580:           tk1 = get_tk
2581:           unless tk1.kind_of?(TkCOMMA) 
2582:             unget_tk tk1
2583:             break
2584:           end
2585:           
2586:           skip_tkspace_comment
2587:           if tk = parse_symbol_in_arg
2588:             args.push tk
2589:             break if no and args.size >= no
2590:           end
2591:         end
2592:       end
2593:       args
2594:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2596
2596:     def parse_symbol_in_arg
2597:       case tk = get_tk
2598:       when TkSYMBOL
2599:         tk.text.sub(/^:/, '')
2600:       when TkSTRING
2601:         eval @read[-1]
2602:       else
2603:         warn("Expected symbol or string, got #{tk.inspect}") if $DEBUG
2604:         nil
2605:       end
2606:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1570
1570:     def parse_toplevel_statements(container)
1571:       comment = collect_first_comment
1572:       look_for_directives_in(container, comment)
1573:       container.comment = comment unless comment.empty?
1574:       parse_statements(container, NORMAL, nil, comment)
1575:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2482
2482:     def parse_visibility(container, single, tk)
2483:       singleton = (single == SINGLE)
2484:       vis = case tk.name
2485:             when "private"   then :private
2486:             when "protected" then :protected
2487:             when "public"    then :public
2488:             when "private_class_method"
2489:               singleton = true
2490:               :private
2491:             when "public_class_method"
2492:               singleton = true
2493:               :public
2494:             else raise "Invalid visibility: #{tk.name}"
2495:             end
2496:             
2497:       skip_tkspace_comment(false)
2498:       case peek_tk
2499:         # Ryan Davis suggested the extension to ignore modifiers, because he
2500:         # often writes
2501:         #
2502:         #   protected unless $TESTING
2503:         #
2504:       when TkNL, TkUNLESS_MOD, TkIF_MOD
2505: #        error("Missing argument") if singleton        
2506:         container.ongoing_visibility = vis
2507:       else
2508:         args = parse_symbol_arg
2509:         container.set_visibility_for(args, vis, singleton)
2510:       end
2511:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2399
2399:   def parse_yield(context, single, tk, method)
2400:     if method.block_params.nil?
2401:       get_tkread
2402:       @scanner.instance_eval{@continue = false}
2403:       method.block_params = parse_yield_parameters
2404:     end
2405:   end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2395
2395:     def parse_yield_parameters
2396:       parse_method_or_yield_parameters
2397:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1532
1532:     def peek_read
1533:       @read.join('')
1534:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1501
1501:     def peek_tk
1502:       unget_tk(tk = get_tk)
1503:       tk
1504:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1449
1449:     def progress(char)
1450:       unless @options.quiet
1451:         @progress.print(char)
1452:         @progress.flush
1453:       end
1454:     end

Directives are modifier comments that can appear after class, module, or method names. For example

  def fred    # :yields:  a, b

or

  class SM  # :nodoc:

we return the directive name and any parameters as a two element array

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2254
2254:     def read_directive(allowed)
2255:       tk = get_tk
2256:       puts "directive: #{tk.inspect}" if $DEBUG
2257:       result = nil
2258:       if tk.kind_of?(TkCOMMENT) 
2259:         if tk.text =~ /\s*:?(\w+):\s*(.*)/
2260:           directive = $1.downcase
2261:           if allowed.include?(directive)
2262:             result = [directive, $2]
2263:           end
2264:         end
2265:       else
2266:         unget_tk(tk)
2267:       end
2268:       result
2269:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2272
2272:     def read_documentation_modifiers(context, allow)
2273:       dir = read_directive(allow)
2274: 
2275:       case dir[0]
2276: 
2277:       when "notnew", "not_new", "not-new"
2278:         context.dont_rename_initialize = true
2279: 
2280:       when "nodoc"
2281:         context.document_self = false
2282:         if dir[1].downcase == "all"
2283:           context.document_children = false
2284:         end
2285: 
2286:       when "doc"
2287:         context.document_self = true
2288:         context.force_documentation = true
2289: 
2290:       when "yield", "yields"
2291:         unless context.params.nil?
2292:           context.params.sub!(/(,|)\s*&\w+/,'') # remove parameter &proc
2293:         end
2294:         context.block_params = dir[1]
2295: 
2296:       when "arg", "args"
2297:         context.params = dir[1]
2298:       end if dir
2299:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2352
2352:     def remove_private_comments(comment)
2353:       comment.gsub!(/^#--.*?^#\+\+/m, '')
2354:       comment.sub!(/^#--.*/m, '')
2355:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1461
1461:     def remove_token_listener(obj)
1462:       @token_listeners.delete(obj)
1463:     end

skip the var [in] part of a ‘for’ statement

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2079
2079:     def skip_for_variable
2080:       skip_tkspace(false)
2081:       tk = get_tk
2082:       skip_tkspace(false)
2083:       tk = get_tk
2084:       unget_tk(tk) unless tk.kind_of?(TkIN)
2085:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2003
2003:     def skip_method(container)
2004:       meth =  AnyMethod.new("", "anon")
2005:       parse_method_parameters(meth)
2006:       parse_statements(container, false, meth)
2007:     end

while, until, and for have an optional

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2088
2088:     def skip_optional_do_after_expression
2089:       skip_tkspace(false)
2090:       tk = get_tk
2091:       case tk
2092:       when TkLPAREN, TkfLPAREN
2093:         end_token = TkRPAREN
2094:       else
2095:         end_token = TkNL
2096:       end
2097: 
2098:       nest = 0
2099:       @scanner.instance_eval{@continue = false}
2100: 
2101:       loop do
2102:         puts("\nWhile: #{tk}, #{@scanner.continue} " +
2103:           "#{@scanner.lex_state} #{nest}") if $DEBUG
2104:         case tk
2105:         when TkSEMICOLON
2106:           break
2107:         when TkLPAREN, TkfLPAREN
2108:           nest += 1
2109:         when TkDO
2110:           break if nest.zero?
2111:         when end_token
2112:           if end_token == TkRPAREN
2113:             nest -= 1
2114:             break if @scanner.lex_state == EXPR_END and nest.zero?
2115:           else
2116:             break unless @scanner.continue
2117:           end
2118:         end
2119:         tk = get_tk
2120:       end
2121:       skip_tkspace(false)
2122:       if peek_tk.kind_of? TkDO
2123:         get_tk
2124:       end
2125:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1516
1516:     def skip_tkspace(skip_nl = true)
1517:       tokens = []
1518:       while ((tk = get_tk).kind_of?(TkSPACE) ||
1519:              (skip_nl && tk.kind_of?(TkNL)))
1520:         tokens.push tk
1521:       end
1522:       unget_tk(tk)
1523:       tokens
1524:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 2538
2538:     def skip_tkspace_comment(skip_nl = true)
2539:       loop do
2540:         skip_tkspace(skip_nl)
2541:         return unless peek_tk.kind_of? TkCOMMENT
2542:         get_tk
2543:       end
2544:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1506
1506:     def unget_tk(tk)
1507:       @tokens.unshift tk
1508:       @unget_read.unshift @read.pop
1509: 
1510:       # Remove this token from any listeners
1511:       @token_listeners.each do |obj|
1512:         obj.pop_token
1513:       end if @token_listeners
1514:     end

[Source]

      # File lib/rdoc/parsers/parse_rb.rb, line 1437
1437:     def warn(msg)
1438:       return if @options.quiet
1439:       msg = make_message msg
1440:       $stderr.puts msg
1441:     end

[Validate]