ASP.NET实现Hadoop增删改查的示例代码

本文介绍了ASP.NET实现Hadoop 增删改查的示例代码，分享给大家，具体如下：

packages.config

<?xml version="1.0" encoding="utf-8"?>

<packages>

<package id="Microsoft.AspNet.WebApi.Client" version="4.0.20505.0" targetFramework="net46" />

<package id="Microsoft.Data.Edm" version="5.2.0" targetFramework="net46" />

<package id="Microsoft.Data.OData" version="5.2.0" targetFramework="net46" />

<package id="Microsoft.Hadoop.WebClient" version="0.12.5126.42915" targetFramework="net46" />

<package id="Microsoft.Net.Http" version="2.0.20505.0" targetFramework="net46" />

<package id="Microsoft.WindowsAzure.ConfigurationManager" version="1.8.0.0" targetFramework="net46" />

<package id="Newtonsoft.Json" version="4.5.11" targetFramework="net46" />

<package id="System.Spatial" version="5.2.0" targetFramework="net46" />

<package id="WindowsAzure.Storage" version="2.0.4.1" targetFramework="net46" />

</packages>

HDFSAccess.cs

using Microsoft.Hadoop.WebHDFS;

using System;

using System.Collections.Generic;

using System.Linq;

using System.Net.Http;

namespace Physical

{

public sealed class HDFSAccess

{

private readonly WebHDFSClient webHDFSClient;

public HDFSAccess(string uriString, string userName)

{

this.webHDFSClient = new WebHDFSClient(new Uri(uriString), userName);

}

public List<string> GetDirectories(string path)

{

var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;

return directoryStatus.Directories.Select(d => d.PathSuffix).ToList();

}

public List<string> GetFiles(string path)

{

var directoryStatus = this.webHDFSClient.GetDirectoryStatus(path).Result;

return directoryStatus.Files.Select(d => d.PathSuffix).ToList();

}

public bool CreateDirectory(string path)

{

// 传入路径不包含根目录时，预设会在根目录「/」底下

return this.webHDFSClient.CreateDirectory(path).Result;

}

public bool DeleteDirectory(string path)

{

// 传入路径不包含根目录时，预设会在根目录「/」底下

return this.webHDFSClient.DeleteDirectory(path).Result;

}

public string CreateFile(string localFile, string remotePath)

{

// 传入远端路径不包含根目录时，预设会在根目录「/」底下

return this.webHDFSClient.CreateFile(localFile, remotePath).Result;

}

public bool DeleteFile(string path)

{

// 传入路径不包含根目录时，预设会在根目录「/」底下

return this.webHDFSClient.DeleteDirectory(path).Result;

}

public HttpResponseMessage OpenFile(string path)

{

// 传入路径不包含根目录时，预设会在根目录「/」底下

return this.webHDFSClient.OpenFile(path).Result;

}

Program.cs

100

101

102

103

104

105

106

107

108

109

110

111

112

113

114

115

116

117

118

119

120

121

122

123

124

125

126

127

128

129

130

131

132

133

134

135

136

137

138

139

140

141

142

143

144

145

146

147

148

149

150

151

152

153

154

155

156

157

158

159

160

161

162

163

164

165

166

167

168

169

170

171

172

173

174

175

176

177

178

179

180

181

182

183

184

185

186

187

188

189

190

191

192

193

194

195

196

197

198

199

200

201

202

203

204

205

206

207

208

209

using Physical;

using System;

using System.Collections.Generic;

using System.IO;

using System.Linq;

using System.Text;

using System.Threading.Tasks;

namespace MyTest

{

class Program

{

// HDFS cluster 客户端进入端点设定于主机上

// 预设端点：http://[主机名称]:50070

// 预设帐号：Wu

private static HDFSAccess access = new HDFSAccess(@"http://127.0.0.1:50070", "Wu");

static void Main(string[] args)

{

GetDirectoriesTest();

Console.WriteLine("----------------------------------------------");

GetFilesTest();

Console.WriteLine("----------------------------------------------");

DirectoryTest();

Console.WriteLine("----------------------------------------------");

FileTest();

Console.WriteLine("----------------------------------------------");

OpenFileTest();

Console.ReadKey();

}

public void TestCleanup()

{

//取得根目录资料夹

var directories = access.GetDirectories(@"/");

// 移除预设目录：tmp、user 外的目录

foreach (var directory in directories)

{

if ("tmp".Equals(directory) || "user".Equals(directory))

{

continue;

}

else

{

access.DeleteDirectory(directory);

}

// 取得根目录档案

var files = access.GetFiles(@"/");

// 移除所有档案

foreach (var file in files)

{

access.DeleteFile(file);

}

// 移除 OpenFile 转存档案

File.Delete(Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg"));

}

public static void GetDirectoriesTest()

{

// 预设根目录下有两个目录：tmp、user

var expected = new List<string>() { "tmp", "user", };

var actual = access.GetDirectories(@"/");

foreach (var item in actual)

{

Console.WriteLine(item);

}

//_传入根目录_预期回传空集合

public static void GetFilesTest()

{

// 预设根目录下没有档案

var expected = new List<string>();

var actual = access.GetFiles(@"/");

foreach (var item in actual)

{

Console.WriteLine(item);

}

//_建立zzz目录_预期成功_预期根目录下有zzz目录_删除zzz目录_预期成功_预期根目录下无zzz目录

public static void DirectoryTest()

{

var directoryName = "zzz";

// 建立zzz目录

var boolCreateDirectory = access.CreateDirectory(directoryName);

Console.WriteLine("建立zzz目录_预期成功:"+boolCreateDirectory);

// 建立zzz目录_预期成功_预期根目录下有zzz目录

// 预设根目录下有三个目录：tmp、user、zzz

var expectedCreateDirectory = new List<string>() { "tmp", "user", directoryName, };

var actualCreateDirectory = access.GetDirectories(@"/");

foreach (var item in actualCreateDirectory)

{

Console.WriteLine(item);

}

Console.WriteLine("********************************************");

// 删除zzz目录

var boolDeleteDirectory = access.DeleteDirectory(directoryName);

Console.WriteLine("删除zzz目录_预期成功:" + boolDeleteDirectory);

// 删除zzz目录_预期成功_预期根目录下无zzz目录

// 预设根目录下有两个目录：tmp、user

var expectedDeleteDirectory = new List<string>() { "tmp", "user", };

var actualDeleteDirectory = access.GetDirectories(@"/");

foreach (var item in actualDeleteDirectory)

{

Console.WriteLine(item);

}

//_建立Test档案_预期根目录下有Test档案_删除Test档案_预期成功_预期根目录下无Test档案

public static void FileTest()

{

var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");

var remotePath = "Test.jpg";

// 建立Test档案

var boolCreateFile = access.CreateFile(localFile, remotePath);

// 建立Test档案_预期根目录下有Test档案

var expectedCreateFile = new List<string>() { remotePath, };

var actualCreateFile = access.GetFiles(@"/");

foreach (var item in actualCreateFile)

{

Console.WriteLine(item);

}

Console.WriteLine("********************************************");

// 删除Test档案

var boolDeleteFile = access.DeleteDirectory(remotePath);

Console.WriteLine("删除Test档案_预期成功:"+boolDeleteFile);

// 删除Test档案_预期成功_预期根目录下无Test档案

var expectedDeleteFile = new List<string>();

var actualDeleteFile = access.GetFiles(@"/");

foreach (var item in actualDeleteFile)

{

Console.WriteLine(item);

}

//_建立Test档案_预期根目录下有Test档案_取得Test档案_预期成功_预期回传Test档案Stream并转存成功

public static void OpenFileTest()

{

var localFile = Path.Combine(Directory.GetCurrentDirectory(), "TestFolder", "Test.jpg");

var remotePath = "Test.jpg";

var saveFile = Path.Combine(Directory.GetCurrentDirectory(), "Test.jpg");

Console.WriteLine("saveFile:" + saveFile);

Console.WriteLine("********************************************");

// 建立Test档案

var boolCreateFile = access.CreateFile(localFile, remotePath);

// 建立Test档案_预期根目录下有Test档案

var expectedCreateFile = new List<string>() { remotePath, };

var actualCreateFile = access.GetFiles(@"/");

foreach (var item in actualCreateFile)

{

Console.WriteLine(item);

}

Console.WriteLine("********************************************");

// 取得Test档案

var response = access.OpenFile(remotePath);

// 取得Test档案_预期成功

response.EnsureSuccessStatusCode();

// 取得Test档案_预期成功_预期回传Test档案Stream并转存成功

using (var fs = File.Create(saveFile))

{

response.Content.CopyToAsync(fs).Wait();

}

Console.WriteLine(File.Exists(saveFile));

}

运行结果如图:

以上就是本文的全部内容，希望对大家的学习有所帮助，也希望大家多多支持快网idc。

原文链接：http://blog.csdn.net/wulex/article/details/78122095

ASP.NET实现Hadoop增删改查的示例代码

相关文章

快网idc优惠网

相关文章

微信

快网idc优惠网

QQ交流群