• 解决flume监听文件夹,上传文件时出现java.nio.charset.MalformedInputException: Input length = 1


    使用flume监听文件夹报错:
    java.nio.charset.MalformedInputException: Input length = 1
    at java.nio.charset.CoderResult.throwException(CoderResult.java:281) ~[?:1.8.0_151]
    at org.apache.flume.serialization.ResettableFileInputStream.readChar(ResettableFileInputStream.java:282) ~[flume-ng-core-1.10.0.jar:1.10.0]
    at org.apache.flume.serialization.LineDeserializer.readLine(LineDeserializer.java:132) ~[flume-ng-core-1.10.0.jar:1.10.0]
    at org.apache.flume.serialization.LineDeserializer.readEvent(LineDeserializer.java:70) ~[flume-ng-core-1.10.0.jar:1.10.0]
    at org.apache.flume.serialization.LineDeserializer.readEvents(LineDeserializer.java:89) ~[flume-ng-core-1.10.0.jar:1.10.0]
    at org.apache.flume.client.avro.ReliableSpoolingFileEventReader.readDeserializerEvents(ReliableSpoolingFileEventReader.java:426) ~[flume-ng-core-1.10.0.jar:1.10.0]
    at org.apache.flume.client.avro.ReliableSpoolingFileEventReader.readEvents(ReliableSpoolingFileEventReader.java:401) ~[flume-ng-core-1.10.0.jar:1.10.0]
    at org.apache.flume.source.SpoolDirectorySource S p o o l D i r e c t o r y R u n n a b l e . r u n ( S p o o l D i r e c t o r y S o u r c e . j a v a : 263 ) [ f l u m e − n g − c o r e − 1.10.0. j a r : 1.10.0 ] a t j a v a . u t i l . c o n c u r r e n t . E x e c u t o r s SpoolDirectoryRunnable.run(SpoolDirectorySource.java:263) [flume-ng-core-1.10.0.jar:1.10.0] at java.util.concurrent.Executors SpoolDirectoryRunnable.run(SpoolDirectorySource.java:263)[flumengcore1.10.0.jar:1.10.0]atjava.util.concurrent.ExecutorsRunnableAdapter.call(Executors.java:511) [?:1.8.0_151]
    at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [?:1.8.0_151]
    at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access 301 ( S c h e d u l e d T h r e a d P o o l E x e c u t o r . j a v a : 180 ) [ ? : 1.8. 0 1 51 ] a t j a v a . u t i l . c o n c u r r e n t . S c h e d u l e d T h r e a d P o o l E x e c u t o r 301(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_151] at java.util.concurrent.ScheduledThreadPoolExecutor 301(ScheduledThreadPoolExecutor.java:180)[?:1.8.0151]atjava.util.concurrent.ScheduledThreadPoolExecutorScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [?:1.8.0_151]
    at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_151]
    at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_151]
    at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151]
    ^C[root@node1 t3]# flume-ng agent --conf conf/ --name a1 --conf-file flume-file-hdfs.conf
    Info: Including Hadoop libraries found via (/usr/local/hadoop271/bin/hadoop) for HDFS access
    Info: Including Hive libraries found via () for Hive access

    • exec /usr/local/java/jdk1.8.0_151/bin/java -Xmx20m -cp ‘conf/:/opt/flume110/lib/:/usr/local/hadoop271/etc/hadoop:/usr/local/hadoop271/share/hadoop/common/lib/:/usr/local/hadoop271/share/hadoop/common/:/usr/local/hadoop271/share/hadoop/hdfs:/usr/local/hadoop271/share/hadoop/hdfs/lib/:/usr/local/hadoop271/share/hadoop/hdfs/:/usr/local/hadoop271/share/hadoop/yarn/lib/:/usr/local/hadoop271/share/hadoop/yarn/:/usr/local/hadoop271/share/hadoop/mapreduce/lib/:/usr/local/hadoop271/share/hadoop/mapreduce/:/usr/local/hadoop271/contrib/capacity-scheduler/.jar:/lib/*’ -Djava.library.path=:/usr/local/hadoop271/lib/native org.apache.flume.node.Application --name a1 --conf-file flume-file-hdfs.conf
      09:44:27.616 [pool-5-thread-1] ERROR org.apache.flume.source.SpoolDirectorySource - FATAL: Spool Directory source r1: { spoolDir: /tmp/t }: Uncaught exception in SpoolDirectorySource thread. Restart or reconfigure Flume to continue processing.
      java.nio.file.InvalidPathException: Malformed input or input contains unmappable characters: /tmp/t/.flumespool/???.txt.COMPLETED
      at sun.nio.fs.UnixPath.encode(UnixPath.java:147) ~[?:1.8.0_151]
      at sun.nio.fs.UnixPath.(UnixPath.java:71) ~[?:1.8.0_151]
      at sun.nio.fs.UnixFileSystem.getPath(UnixFileSystem.java:281) ~[?:1.8.0_151]
      at java.nio.file.Paths.get(Paths.java:84) ~[?:1.8.0_151]
      at org.apache.flume.client.avro.ReliableSpoolingFileEventReader.isFileInTrackerDir(ReliableSpoolingFileEventReader.java:330) ~[flume-ng-core-1.10.0.jar:1.10.0]
      at org.apache.flume.client.avro.ReliableSpoolingFileEventReader.access$300(ReliableSpoolingFileEventReader.java:92) ~[flume-ng-core-1.10.0.jar:1.10.0]
      at org.apache.flume.client.avro.ReliableSpoolingFileEventReader$1.visitFile(ReliableSpoolingFileEventReader.java:287) ~[flume-ng-core-1.10.0.jar:1.10.0]
      at org.apache.flume.client.avro.ReliableSpoolingFileEventReader 1. v i s i t F i l e ( R e l i a b l e S p o o l i n g F i l e E v e n t R e a d e r . j a v a : 266 )   [ f l u m e − n g − c o r e − 1.10.0. j a r : 1.10.0 ] a t j a v a . n i o . f i l e . F i l e s . w a l k F i l e T r e e ( F i l e s . j a v a : 2670 )   [ ? : 1.8. 0 1 51 ] a t j a v a . n i o . f i l e . F i l e s . w a l k F i l e T r e e ( F i l e s . j a v a : 2742 )   [ ? : 1.8. 0 1 51 ] a t o r g . a p a c h e . f l u m e . c l i e n t . a v r o . R e l i a b l e S p o o l i n g F i l e E v e n t R e a d e r . g e t C a n d i d a t e F i l e s ( R e l i a b l e S p o o l i n g F i l e E v e n t R e a d e r . j a v a : 266 )   [ f l u m e − n g − c o r e − 1.10.0. j a r : 1.10.0 ] a t o r g . a p a c h e . f l u m e . c l i e n t . a v r o . R e l i a b l e S p o o l i n g F i l e E v e n t R e a d e r . g e t N e x t F i l e ( R e l i a b l e S p o o l i n g F i l e E v e n t R e a d e r . j a v a : 629 )   [ f l u m e − n g − c o r e − 1.10.0. j a r : 1.10.0 ] a t o r g . a p a c h e . f l u m e . c l i e n t . a v r o . R e l i a b l e S p o o l i n g F i l e E v e n t R e a d e r . r e a d E v e n t s ( R e l i a b l e S p o o l i n g F i l e E v e n t R e a d e r . j a v a : 393 )   [ f l u m e − n g − c o r e − 1.10.0. j a r : 1.10.0 ] a t o r g . a p a c h e . f l u m e . s o u r c e . S p o o l D i r e c t o r y S o u r c e 1.visitFile(ReliableSpoolingFileEventReader.java:266) ~[flume-ng-core-1.10.0.jar:1.10.0] at java.nio.file.Files.walkFileTree(Files.java:2670) ~[?:1.8.0_151] at java.nio.file.Files.walkFileTree(Files.java:2742) ~[?:1.8.0_151] at org.apache.flume.client.avro.ReliableSpoolingFileEventReader.getCandidateFiles(ReliableSpoolingFileEventReader.java:266) ~[flume-ng-core-1.10.0.jar:1.10.0] at org.apache.flume.client.avro.ReliableSpoolingFileEventReader.getNextFile(ReliableSpoolingFileEventReader.java:629) ~[flume-ng-core-1.10.0.jar:1.10.0] at org.apache.flume.client.avro.ReliableSpoolingFileEventReader.readEvents(ReliableSpoolingFileEventReader.java:393) ~[flume-ng-core-1.10.0.jar:1.10.0] at org.apache.flume.source.SpoolDirectorySource 1.visitFile(ReliableSpoolingFileEventReader.java:266) [flumengcore1.10.0.jar:1.10.0]atjava.nio.file.Files.walkFileTree(Files.java:2670) [?:1.8.0151]atjava.nio.file.Files.walkFileTree(Files.java:2742) [?:1.8.0151]atorg.apache.flume.client.avro.ReliableSpoolingFileEventReader.getCandidateFiles(ReliableSpoolingFileEventReader.java:266) [flumengcore1.10.0.jar:1.10.0]atorg.apache.flume.client.avro.ReliableSpoolingFileEventReader.getNextFile(ReliableSpoolingFileEventReader.java:629) [flumengcore1.10.0.jar:1.10.0]atorg.apache.flume.client.avro.ReliableSpoolingFileEventReader.readEvents(ReliableSpoolingFileEventReader.java:393) [flumengcore1.10.0.jar:1.10.0]atorg.apache.flume.source.SpoolDirectorySourceSpoolDirectoryRunnable.run(SpoolDirectorySource.java:263) [flume-ng-core-1.10.0.jar:1.10.0]
      at java.util.concurrent.Executors R u n n a b l e A d a p t e r . c a l l ( E x e c u t o r s . j a v a : 511 ) [ ? : 1.8. 0 1 51 ] a t j a v a . u t i l . c o n c u r r e n t . F u t u r e T a s k . r u n A n d R e s e t ( F u t u r e T a s k . j a v a : 308 ) [ ? : 1.8. 0 1 51 ] a t j a v a . u t i l . c o n c u r r e n t . S c h e d u l e d T h r e a d P o o l E x e c u t o r RunnableAdapter.call(Executors.java:511) [?:1.8.0_151] at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308) [?:1.8.0_151] at java.util.concurrent.ScheduledThreadPoolExecutor RunnableAdapter.call(Executors.java:511)[?:1.8.0151]atjava.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)[?:1.8.0151]atjava.util.concurrent.ScheduledThreadPoolExecutorScheduledFutureTask.access 301 ( S c h e d u l e d T h r e a d P o o l E x e c u t o r . j a v a : 180 ) [ ? : 1.8. 0 1 51 ] a t j a v a . u t i l . c o n c u r r e n t . S c h e d u l e d T h r e a d P o o l E x e c u t o r 301(ScheduledThreadPoolExecutor.java:180) [?:1.8.0_151] at java.util.concurrent.ScheduledThreadPoolExecutor 301(ScheduledThreadPoolExecutor.java:180)[?:1.8.0151]atjava.util.concurrent.ScheduledThreadPoolExecutorScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294) [?:1.8.0_151]
      at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_151]
      at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_151]
      at java.lang.Thread.run(Thread.java:748) [?:1.8.0_151]
      在这里插入图片描述
      在这里插入图片描述
      解决: 上传文件名不要有中文,上传.txt .log文件,一定要是日志文本可读的文件。 因为我上传了exe,然后又上传了 中文名的文件,就报错了。
      Flume是海量日志采集工具
       最后上传一个文本就好了
      在这里插入图片描述
  • 相关阅读:
    【详细学习SpringBoot自动装配原理分析之核心流程初解析-1】
    HDMI简单介绍
    初次邂逅 EasyExcel
    SpringBoot解决LocalDateTime返回数据为数组问题
    Spring Boot面试题
    操作系统-进程与线程(同步互斥典型模型-生产者,消费者模型、吸烟者问题)
    Java-数据结构-链表<三>
    Java -- 每日一问:Java常见的垃圾收集器有哪些?
    Go 使用 RabbitMQ---------------之一
    包装行业B2B电商平台:实现供需调配和精准对接,提高企业经济效益
  • 原文地址:https://blog.csdn.net/wasane/article/details/125439953