查看: 1781|回复: 0

[ASP.NET教程] ASP.NET实现Hadoop增删改查的示例代码

发表于 2018-1-3 08:00:01

本文介绍了asp.net实现Hadoop增删改查的示例代码,分享给大家,具体如下:

packages.config

  1. <?xml version="1.0" encoding="utf-8"?>
  2. <packages>
  3. <package id="Microsoft.AspNet.WebApi.Client" version="4.0.20505.0" targetFramework="net46" />
  4. <package id="Microsoft.Data.Edm" version="5.2.0" targetFramework="net46" />
  5. <package id="Microsoft.Data.OData" version="5.2.0" targetFramework="net46" />
  6. <package id="Microsoft.Hadoop.WebClient" version="0.12.5126.42915" targetFramework="net46" />
  7. <package id="Microsoft.Net.Http" version="2.0.20505.0" targetFramework="net46" />
  8. <package id="Microsoft.WindowsAzure.ConfigurationManager" version="1.8.0.0" targetFramework="net46" />
  9. <package id="Newtonsoft.Json" version="4.5.11" targetFramework="net46" />
  10. <package id="System.Spatial" version="5.2.0" targetFramework="net46" />
  11. <package id="WindowsAzure.Storage" version="2.0.4.1" targetFramework="net46" />
  12. </packages>
复制代码

HDFSAccess.cs

  1. using Microsoft.Hadoop.WebHDFS;
  2. using System;
  3. using System.Collections.Generic;
  4. using System.Linq;
  5. using System.Net.Http;
  6. namespace Physical
  7. {
  8. public sealed class HDFSAccess
  9. {
  10. private readonly WebHDFSClient webHDFSClient;
  11. public HDFSAccess(string uriString, string userName)
  12. {
  13. this.webHDFSClient = new WebHDFSClient(new Uri(uriString), userName);
  14. }
  15. public List<string> GetDirectories(string path)
  16. {
  17. var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;
  18. return directoryStatus.Directories.Select(d => d.PathSuffix).ToList();
  19. }
  20. public List<string> GetFiles(string path)
  21. {
  22. var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;
  23. return directoryStatus.Files.Select(d => d.PathSuffix).ToList();
  24. }
  25. public bool CreateDirectory(string path)
  26. {
  27. // 传入路径不包含根目录时,预设会在根目录「/」底下
  28. return this.webHDFSClient.CreateDirectory(path).Result;
  29. }
  30. public bool DeleteDirectory(string path)
  31. {
  32. // 传入路径不包含根目录时,预设会在根目录「/」底下
  33. return this.webHDFSClient.DeleteDirectory(path).Result;
  34. }
  35. public string CreateFile(string localFile, string remotePath)
  36. {
  37. // 传入远端路径不包含根目录时,预设会在根目录「/」底下
  38. return this.webHDFSClient.CreateFile(localFile, remotePath).Result;
  39. }
  40. public bool DeleteFile(string path)
  41. {
  42. // 传入路径不包含根目录时,预设会在根目录「/」底下
  43. return this.webHDFSClient.DeleteDirectory(path).Result;
  44. }
  45. public HttpResponseMessage OpenFile(string path)
  46. {
  47. // 传入路径不包含根目录时,预设会在根目录「/」底下
  48. return this.webHDFSClient.OpenFile(path).Result;
  49. }
  50. }
  51. }
复制代码

Program.cs

  1. using Physical;
  2. using System;
  3. using System.Collections.Generic;
  4. using System.IO;
  5. using System.Linq;
  6. using System.Text;
  7. using System.Threading.Tasks;
  8. namespace MyTest
  9. {
  10. class Program
  11. {
  12. // HDFS cluster 客户端进入端点设定于 主机上
  13. // 预设端点:http://[主机名称]:50070
  14. // 预设帐号:Wu
  15. private static HDFSAccess access = new HDFSAccess(@"http://127.0.0.1:50070", "Wu");
  16. static void Main(string[] args)
  17. {
  18. GetDirectoriesTest();
  19. Console.WriteLine("----------------------------------------------");
  20. GetFilesTest();
  21. Console.WriteLine("----------------------------------------------");
  22. DirectoryTest();
  23. Console.WriteLine("----------------------------------------------");
  24. FileTest();
  25. Console.WriteLine("----------------------------------------------");
  26. OpenFileTest();
  27. Console.ReadKey();
  28. }
  29. public void TestCleanup()
  30. {
  31. //取得根目录资料夹
  32. var directories = access.GetDirectories(@"/");
  33. // 移除预设目录:tmp、user 外的目录
  34. foreach (var directory in directories)
  35. {
  36. if ("tmp".Equals(directory) || "user".Equals(directory))
  37. {
  38. continue;
  39. }
  40. else
  41. {
  42. access.DeleteDirectory(directory);
  43. }
  44. }
  45. // 取得根目录档案
  46. var files = access.GetFiles(@"/");
  47. // 移除所有档案
  48. foreach (var file in files)
  49. {
  50. access.DeleteFile(file);
  51. }
  52. // 移除 OpenFile 转存档案
  53. File.Delete(Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg"));
  54. }
  55. //_传入根目录_预期回传预设目录
  56. public static void GetDirectoriesTest()
  57. {
  58. // 预设根目录下有两个目录:tmp、user
  59. var expected = new List<string>() { "tmp", "user", };
  60. var actual = access.GetDirectories(@"/");
  61. foreach (var item in actual)
  62. {
  63. Console.WriteLine(item);
  64. }
  65. }
  66. //_传入根目录_预期回传空集合
  67. public static void GetFilesTest()
  68. {
  69. // 预设根目录下没有档案
  70. var expected = new List<string>();
  71. var actual = access.GetFiles(@"/");
  72. foreach (var item in actual)
  73. {
  74. Console.WriteLine(item);
  75. }
  76. }
  77. //_建立zzz目录_预期成功_预期根目录下有zzz目录_删除zzz目录_预期成功_预期根目录下无zzz目录
  78. public static void DirectoryTest()
  79. {
  80. var directoryName = "zzz";
  81. // 建立zzz目录
  82. var boolCreateDirectory = access.CreateDirectory(directoryName);
  83. Console.WriteLine("建立zzz目录_预期成功:"+boolCreateDirectory);
  84. // 建立zzz目录_预期成功_预期根目录下有zzz目录
  85. // 预设根目录下有三个目录:tmp、user、zzz
  86. var expectedCreateDirectory = new List<string>() { "tmp", "user", directoryName, };
  87. var actualCreateDirectory = access.GetDirectories(@"/");
  88. foreach (var item in actualCreateDirectory)
  89. {
  90. Console.WriteLine(item);
  91. }
  92. Console.WriteLine("********************************************");
  93. // 删除zzz目录
  94. var boolDeleteDirectory = access.DeleteDirectory(directoryName);
  95. Console.WriteLine("删除zzz目录_预期成功:" + boolDeleteDirectory);
  96. // 删除zzz目录_预期成功_预期根目录下无zzz目录
  97. // 预设根目录下有两个目录:tmp、user
  98. var expectedDeleteDirectory = new List<string>() { "tmp", "user", };
  99. var actualDeleteDirectory = access.GetDirectories(@"/");
  100. foreach (var item in actualDeleteDirectory)
  101. {
  102. Console.WriteLine(item);
  103. }
  104. }
  105. //_建立Test档案_预期根目录下有Test档案_删除Test档案_预期成功_预期根目录下无Test档案
  106. public static void FileTest()
  107. {
  108. var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");
  109. var remotePath = "Test.jpg";
  110. // 建立Test档案
  111. var boolCreateFile = access.CreateFile(localFile, remotePath);
  112. // 建立Test档案_预期根目录下有Test档案
  113. var expectedCreateFile = new List<string>() { remotePath, };
  114. var actualCreateFile = access.GetFiles(@"/");
  115. foreach (var item in actualCreateFile)
  116. {
  117. Console.WriteLine(item);
  118. }
  119. Console.WriteLine("********************************************");
  120. // 删除Test档案
  121. var boolDeleteFile = access.DeleteDirectory(remotePath);
  122. Console.WriteLine("删除Test档案_预期成功:"+boolDeleteFile);
  123. // 删除Test档案_预期成功_预期根目录下无Test档案
  124. var expectedDeleteFile = new List<string>();
  125. var actualDeleteFile = access.GetFiles(@"/");
  126. foreach (var item in actualDeleteFile)
  127. {
  128. Console.WriteLine(item);
  129. }
  130. }
  131. //_建立Test档案_预期根目录下有Test档案_取得Test档案_预期成功_预期回传Test档案Stream并转存成功
  132. public static void OpenFileTest()
  133. {
  134. var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");
  135. var remotePath = "Test.jpg";
  136. var saveFile = Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg");
  137. Console.WriteLine("saveFile:" + saveFile);
  138. Console.WriteLine("********************************************");
  139. // 建立Test档案
  140. var boolCreateFile = access.CreateFile(localFile, remotePath);
  141. // 建立Test档案_预期根目录下有Test档案
  142. var expectedCreateFile = new List<string>() { remotePath, };
  143. var actualCreateFile = access.GetFiles(@"/");
  144. foreach (var item in actualCreateFile)
  145. {
  146. Console.WriteLine(item);
  147. }
  148. Console.WriteLine("********************************************");
  149. // 取得Test档案
  150. var response = access.OpenFile(remotePath);
  151. // 取得Test档案_预期成功
  152. response.EnsureSuccessStatusCode();
  153. // 取得Test档案_预期成功_预期回传Test档案Stream并转存成功
  154. using (var fs = File.Create(saveFile))
  155. {
  156. response.Content.CopyToAsync(fs).Wait();
  157. }
  158. Console.WriteLine(File.Exists(saveFile));
  159. }
  160. }
  161. }
复制代码

运行结果如图:

以上就是本文的全部内容,希望对大家的学习有所帮助,也希望大家多多支持程序员之家。



回复

使用道具 举报