Hot-keys on this page

r m x p   toggle line displays

j k   next/prev highlighted chunk

0   (zero) top of page

1   (one) first highlighted chunk

1

2

3

4

5

6

7

8

9

10

11

12

13

14

15

16

17

18

19

20

21

22

23

24

25

26

27

28

29

30

31

32

33

34

35

36

37

38

39

40

41

42

43

44

45

46

47

48

49

50

51

52

53

54

55

56

57

58

59

60

61

62

63

64

65

66

67

68

69

70

71

72

73

74

75

76

77

78

79

80

81

82

83

84

85

86

87

88

89

90

91

92

93

94

95

96

97

98

99

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

# -*- coding: utf-8 -*- 

""" 

    pygments.formatters.other 

    ~~~~~~~~~~~~~~~~~~~~~~~~~ 

 

    Other formatters: NullFormatter, RawTokenFormatter. 

 

    :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. 

    :license: BSD, see LICENSE for details. 

""" 

 

from pygments.formatter import Formatter 

from pygments.util import OptionError, get_choice_opt 

from pygments.token import Token 

from pygments.console import colorize 

 

__all__ = ['NullFormatter', 'RawTokenFormatter', 'TestcaseFormatter'] 

 

 

class NullFormatter(Formatter): 

    """ 

    Output the text unchanged without any formatting. 

    """ 

    name = 'Text only' 

    aliases = ['text', 'null'] 

    filenames = ['*.txt'] 

 

    def format(self, tokensource, outfile): 

        enc = self.encoding 

        for ttype, value in tokensource: 

            if enc: 

                outfile.write(value.encode(enc)) 

            else: 

                outfile.write(value) 

 

 

class RawTokenFormatter(Formatter): 

    r""" 

    Format tokens as a raw representation for storing token streams. 

 

    The format is ``tokentype<TAB>repr(tokenstring)\n``. The output can later 

    be converted to a token stream with the `RawTokenLexer`, described in the 

    :doc:`lexer list <lexers>`. 

 

    Only two options are accepted: 

 

    `compress` 

        If set to ``'gz'`` or ``'bz2'``, compress the output with the given 

        compression algorithm after encoding (default: ``''``). 

    `error_color` 

        If set to a color name, highlight error tokens using that color.  If 

        set but with no value, defaults to ``'red'``. 

 

        .. versionadded:: 0.11 

 

    """ 

    name = 'Raw tokens' 

    aliases = ['raw', 'tokens'] 

    filenames = ['*.raw'] 

 

    unicodeoutput = False 

 

    def __init__(self, **options): 

        Formatter.__init__(self, **options) 

        # We ignore self.encoding if it is set, since it gets set for lexer 

        # and formatter if given with -Oencoding on the command line. 

        # The RawTokenFormatter outputs only ASCII. Override here. 

        self.encoding = 'ascii'  # let pygments.format() do the right thing 

        self.compress = get_choice_opt(options, 'compress', 

                                       ['', 'none', 'gz', 'bz2'], '') 

        self.error_color = options.get('error_color', None) 

        if self.error_color is True: 

            self.error_color = 'red' 

        if self.error_color is not None: 

            try: 

                colorize(self.error_color, '') 

            except KeyError: 

                raise ValueError("Invalid color %r specified" % 

                                 self.error_color) 

 

    def format(self, tokensource, outfile): 

        try: 

            outfile.write(b'') 

        except TypeError: 

            raise TypeError('The raw tokens formatter needs a binary ' 

                            'output file') 

        if self.compress == 'gz': 

            import gzip 

            outfile = gzip.GzipFile('', 'wb', 9, outfile) 

            def write(text): 

                outfile.write(text.encode()) 

            flush = outfile.flush 

        elif self.compress == 'bz2': 

            import bz2 

            compressor = bz2.BZ2Compressor(9) 

            def write(text): 

                outfile.write(compressor.compress(text.encode())) 

            def flush(): 

                outfile.write(compressor.flush()) 

                outfile.flush() 

        else: 

            def write(text): 

                outfile.write(text.encode()) 

            flush = outfile.flush 

 

        if self.error_color: 

            for ttype, value in tokensource: 

                line = "%s\t%r\n" % (ttype, value) 

                if ttype is Token.Error: 

                    write(colorize(self.error_color, line)) 

                else: 

                    write(line) 

        else: 

            for ttype, value in tokensource: 

                write("%s\t%r\n" % (ttype, value)) 

        flush() 

 

TESTCASE_BEFORE = u'''\ 

    def testNeedsName(self): 

        fragment = %r 

        tokens = [ 

''' 

TESTCASE_AFTER = u'''\ 

        ] 

        self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) 

''' 

 

 

class TestcaseFormatter(Formatter): 

    """ 

    Format tokens as appropriate for a new testcase. 

 

    .. versionadded:: 2.0 

    """ 

    name = 'Testcase' 

    aliases = ['testcase'] 

 

    def __init__(self, **options): 

        Formatter.__init__(self, **options) 

        if self.encoding is not None and self.encoding != 'utf-8': 

            raise ValueError("Only None and utf-8 are allowed encodings.") 

 

    def format(self, tokensource, outfile): 

        indentation = ' ' * 12 

        rawbuf = [] 

        outbuf = [] 

        for ttype, value in tokensource: 

            rawbuf.append(value) 

            outbuf.append('%s(%s, %r),\n' % (indentation, ttype, value)) 

 

        before = TESTCASE_BEFORE % (u''.join(rawbuf),) 

        during = u''.join(outbuf) 

        after = TESTCASE_AFTER 

        if self.encoding is None: 

            outfile.write(before + during + after) 

        else: 

            outfile.write(before.encode('utf-8')) 

            outfile.write(during.encode('utf-8')) 

            outfile.write(after.encode('utf-8')) 

        outfile.flush()